pax_global_header00006660000000000000000000000064145652013500014513gustar00rootroot0000000000000052 comment=5c0bf7788430ac5d4a1d053455028f1bf284e1d9 laniakea-spark-0.1.1/000077500000000000000000000000001456520135000143755ustar00rootroot00000000000000laniakea-spark-0.1.1/.github/000077500000000000000000000000001456520135000157355ustar00rootroot00000000000000laniakea-spark-0.1.1/.github/workflows/000077500000000000000000000000001456520135000177725ustar00rootroot00000000000000laniakea-spark-0.1.1/.github/workflows/python.yml000066400000000000000000000024311456520135000220360ustar00rootroot00000000000000name: Build & Test on: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: ubuntu-latest strategy: matrix: python-version: [ '3.11' ] name: Python ${{ matrix.python-version }} steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Update cache run: sudo apt-get update -qq - name: Install system prerequisites run: sudo apt-get install -yq debspawn - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install flake8 pytest pylint mypy black isort - name: Build & Install run: | ./setup.py build ./setup.py install --root=/tmp rm -rf build/ - name: Lint (flake8) run: | python -m flake8 ./ --statistics - name: Lint (pylint) run: | python -m pylint -f colorized ./spark/ - name: Lint (mypy) run: | python -m mypy . - name: Style check (isort) run: | python -m isort --diff . - name: Style check (black) run: | python -m black --diff . laniakea-spark-0.1.1/.gitignore000066400000000000000000000026551456520135000163750ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ parts/ sdist/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # IPython profile_default/ ipython_config.py # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # PyCharm project settings .idea # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ laniakea-spark-0.1.1/.mypy.ini000066400000000000000000000002761456520135000161570ustar00rootroot00000000000000[mypy] show_column_numbers = True pretty = True strict_optional = False ignore_missing_imports = True warn_redundant_casts = True warn_unused_ignores = True laniakea-spark-0.1.1/AUTHORS000066400000000000000000000000501456520135000154400ustar00rootroot00000000000000Matthias Klumpp laniakea-spark-0.1.1/COPYING000066400000000000000000000167431456520135000154430ustar00rootroot00000000000000 GNU LESSER GENERAL PUBLIC LICENSE Version 3, 29 June 2007 Copyright (C) 2007 Free Software Foundation, Inc. Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. This version of the GNU Lesser General Public License incorporates the terms and conditions of version 3 of the GNU General Public License, supplemented by the additional permissions listed below. 0. Additional Definitions. As used herein, "this License" refers to version 3 of the GNU Lesser General Public License, and the "GNU GPL" refers to version 3 of the GNU General Public License. "The Library" refers to a covered work governed by this License, other than an Application or a Combined Work as defined below. An "Application" is any work that makes use of an interface provided by the Library, but which is not otherwise based on the Library. Defining a subclass of a class defined by the Library is deemed a mode of using an interface provided by the Library. A "Combined Work" is a work produced by combining or linking an Application with the Library. The particular version of the Library with which the Combined Work was made is also called the "Linked Version". The "Minimal Corresponding Source" for a Combined Work means the Corresponding Source for the Combined Work, excluding any source code for portions of the Combined Work that, considered in isolation, are based on the Application, and not on the Linked Version. The "Corresponding Application Code" for a Combined Work means the object code and/or source code for the Application, including any data and utility programs needed for reproducing the Combined Work from the Application, but excluding the System Libraries of the Combined Work. 1. Exception to Section 3 of the GNU GPL. You may convey a covered work under sections 3 and 4 of this License without being bound by section 3 of the GNU GPL. 2. Conveying Modified Versions. If you modify a copy of the Library, and, in your modifications, a facility refers to a function or data to be supplied by an Application that uses the facility (other than as an argument passed when the facility is invoked), then you may convey a copy of the modified version: a) under this License, provided that you make a good faith effort to ensure that, in the event an Application does not supply the function or data, the facility still operates, and performs whatever part of its purpose remains meaningful, or b) under the GNU GPL, with none of the additional permissions of this License applicable to that copy. 3. Object Code Incorporating Material from Library Header Files. The object code form of an Application may incorporate material from a header file that is part of the Library. You may convey such object code under terms of your choice, provided that, if the incorporated material is not limited to numerical parameters, data structure layouts and accessors, or small macros, inline functions and templates (ten or fewer lines in length), you do both of the following: a) Give prominent notice with each copy of the object code that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the object code with a copy of the GNU GPL and this license document. 4. Combined Works. You may convey a Combined Work under terms of your choice that, taken together, effectively do not restrict modification of the portions of the Library contained in the Combined Work and reverse engineering for debugging such modifications, if you also do each of the following: a) Give prominent notice with each copy of the Combined Work that the Library is used in it and that the Library and its use are covered by this License. b) Accompany the Combined Work with a copy of the GNU GPL and this license document. c) For a Combined Work that displays copyright notices during execution, include the copyright notice for the Library among these notices, as well as a reference directing the user to the copies of the GNU GPL and this license document. d) Do one of the following: 0) Convey the Minimal Corresponding Source under the terms of this License, and the Corresponding Application Code in a form suitable for, and under terms that permit, the user to recombine or relink the Application with a modified version of the Linked Version to produce a modified Combined Work, in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source. 1) Use a suitable shared library mechanism for linking with the Library. A suitable mechanism is one that (a) uses at run time a copy of the Library already present on the user's computer system, and (b) will operate properly with a modified version of the Library that is interface-compatible with the Linked Version. e) Provide Installation Information, but only if you would otherwise be required to provide such information under section 6 of the GNU GPL, and only to the extent that such information is necessary to install and execute a modified version of the Combined Work produced by recombining or relinking the Application with a modified version of the Linked Version. (If you use option 4d0, the Installation Information must accompany the Minimal Corresponding Source and Corresponding Application Code. If you use option 4d1, you must provide the Installation Information in the manner specified by section 6 of the GNU GPL for conveying Corresponding Source.) 5. Combined Libraries. You may place library facilities that are a work based on the Library side by side in a single library together with other library facilities that are not Applications and are not covered by this License, and convey such a combined library under terms of your choice, if you do both of the following: a) Accompany the combined library with a copy of the same work based on the Library, uncombined with any other library facilities, conveyed under the terms of this License. b) Give prominent notice with the combined library that part of it is a work based on the Library, and explaining where to find the accompanying uncombined form of the same work. 6. Revised Versions of the GNU Lesser General Public License. The Free Software Foundation may publish revised and/or new versions of the GNU Lesser General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Library as you received it specifies that a certain numbered version of the GNU Lesser General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that published version or of any later version published by the Free Software Foundation. If the Library as you received it does not specify a version number of the GNU Lesser General Public License, you may choose any version of the GNU Lesser General Public License ever published by the Free Software Foundation. If the Library as you received it specifies that a proxy can decide whether future versions of the GNU Lesser General Public License shall apply, that proxy's public statement of acceptance of any version is permanent authorization for you to choose that version for the Library. laniakea-spark-0.1.1/README.md000066400000000000000000000021521456520135000156540ustar00rootroot00000000000000# Laniakea Spark [![Build & Test](https://github.com/lkhq/laniakea-spark/actions/workflows/python.yml/badge.svg)](https://github.com/lkhq/laniakea-spark/actions/workflows/python.yml) Spark is the generic Laniakea job runner and package build executor. It is able to perform a variety of tasks on dedicated builder machines, like building packages, distribution ISO images or performing longer QA tasks. Spark instances communicate with Lighthouse servers via ZeroMQ to fetch new jobs and report information. They auto-register with the master system, if they were provided with the right credentials for the respective instance. ## Setup Instructions Minimum required Debian release: 11.0 (Bullseye) ### Dependencies ```Bash sudo apt install \ python3-debian \ python3-zmq \ python3-setuptools \ python3-firehose \ gnupg \ dput-ng \ debspawn ``` ### Installation You can find more information on how to set up Spark instances at [the Laniakea documentation](https://laniakea-hq.readthedocs.io/latest/general/worker-setup.html) or check out our [Ansible provisioning templates](https://github.com/lkhq/spark-setup). laniakea-spark-0.1.1/RELEASE000066400000000000000000000005471456520135000154060ustar00rootroot00000000000000Laniakea-Spark Release Notes 1. Tag release in Git: git tag -s -f -m "Release 0.1.1" v0.1.1 git push --tags git push 2. Upload to PyPI: python setup.py sdist twine upload dist/* 3. Do post release version bump in `RELEASE` and `spark/__init__.py` 4. Commit trivial changes: git commit -a -m "trivial: post release version bump" git push laniakea-spark-0.1.1/data/000077500000000000000000000000001456520135000153065ustar00rootroot00000000000000laniakea-spark-0.1.1/data/sudo/000077500000000000000000000000001456520135000162605ustar00rootroot00000000000000laniakea-spark-0.1.1/data/sudo/10laniakea-spark000066400000000000000000000000631456520135000212260ustar00rootroot00000000000000_lkspark ALL=(ALL) NOPASSWD: @PREFIX@/bin/debspawn laniakea-spark-0.1.1/data/systemd/000077500000000000000000000000001456520135000167765ustar00rootroot00000000000000laniakea-spark-0.1.1/data/systemd/laniakea-spark.service.in000066400000000000000000000003761456520135000236560ustar00rootroot00000000000000[Unit] Description=Laniakea Spark After=syslog.target network.target ConditionPathExists=/etc/laniakea/spark.toml [Service] Type=simple Restart=on-failure User=_lkspark Group=nogroup ExecStart=@PREFIX@/bin/lk-spark [Install] WantedBy=multi-user.target laniakea-spark-0.1.1/data/ws/000077500000000000000000000000001456520135000157375ustar00rootroot00000000000000laniakea-spark-0.1.1/data/ws/README000066400000000000000000000000021456520135000166070ustar00rootroot00000000000000 laniakea-spark-0.1.1/install-sysdata.py000077500000000000000000000066501456520135000200750ustar00rootroot00000000000000#!/usr/bin/env python3 # -*- coding: utf-8 -*- # # Copyright (C) 2018-2024 Matthias Klumpp # # SPDX-License-Identifier: LGPL-3.0-or-later # # This is a helper script to install additional configuration and documentation into # system locations, which Python's setuptools and pip will not usually let us install. # import os import sys import shutil from pathlib import Path from argparse import ArgumentParser from tempfile import TemporaryDirectory try: import pkgconfig except ImportError: print() print( ( 'Unable to import pkgconfig. Please install the module ' '(apt install python3-pkgconfig or pip install pkgconfig) ' 'to continue.' ) ) print() sys.exit(4) class Installer: def __init__(self, root: str = None, prefix: str = None): if not root: root = os.environ.get('DESTDIR') if not root: root = '/' self.root = root if not prefix: prefix = '/usr/local' if self.root == '/' else '/usr' if prefix.startswith('/'): prefix = prefix[1:] self.prefix = prefix def install(self, src, dst, replace_vars=False): if dst.startswith('/'): dst = dst[1:] dst_full = os.path.join(self.root, dst, os.path.basename(src)) else: dst_full = os.path.join(self.root, self.prefix, dst, os.path.basename(src)) if dst_full.endswith('.in'): dst_full = dst_full[:-3] Path(os.path.dirname(dst_full)).mkdir(mode=0o755, parents=True, exist_ok=True) if replace_vars: with open(src, 'r') as f_src: with open(dst_full, 'w') as f_dst: for line in f_src: f_dst.write(line.replace('@PREFIX@', '/' + self.prefix)) else: shutil.copy(src, dst_full) os.chmod(dst_full, 0o644) print('{}\t\t{}'.format(os.path.basename(src), dst_full)) def chdir_to_source_root(): thisfile = __file__ if not os.path.isabs(thisfile): thisfile = os.path.normpath(os.path.join(os.getcwd(), thisfile)) os.chdir(os.path.dirname(thisfile)) def install_data(temp_dir: str, root_dir: str, prefix_dir: str): chdir_to_source_root() print('Checking dependencies') if not pkgconfig.installed('systemd', '>= 240'): print('Systemd is not installed on this system. Please make systemd available to continue.') sys.exit(4) print('Installing data') inst = Installer(root_dir, prefix_dir) sd_system_unit_dir = pkgconfig.variables('systemd')['systemdsystemunitdir'] inst.install('data/systemd/laniakea-spark.service.in', sd_system_unit_dir, replace_vars=True) inst.install('data/sudo/10laniakea-spark', '/etc/sudoers.d/', replace_vars=True) def main(): parser = ArgumentParser(description='Debspawn system data installer') parser.add_argument( '--root', action='store', dest='root', default=None, help='Root directory to install into.' ) parser.add_argument( '--prefix', action='store', dest='prefix', default=None, help='Directory prefix (usually `/usr` or `/usr/local`).', ) options = parser.parse_args(sys.argv[1:]) with TemporaryDirectory(prefix='dsinstall-') as temp_dir: install_data(temp_dir, options.root, options.prefix) return 0 if __name__ == '__main__': sys.exit(main()) laniakea-spark-0.1.1/lint-and-format.sh000077500000000000000000000004561456520135000177350ustar00rootroot00000000000000#!/usr/bin/env bash set -e BASEDIR=$(dirname "$0") cd $BASEDIR echo "=== ISort ===" isort . echo "=== Black ===" black . echo "=== Flake8 ===" python -m flake8 ./ --statistics echo "✓" echo "=== Pylint ===" python -m pylint -f colorized ./spark/ echo "✓" echo "=== MyPy ===" python -m mypy . laniakea-spark-0.1.1/pyproject.toml000066400000000000000000000021741456520135000173150ustar00rootroot00000000000000[project] name = "laniakea-spark" description = "Generic distributed job runner for Laniakea." authors = [ {name = "Matthias Klumpp", email = "matthias@tenstral.net"}, ] license = {text="LGPL-3.0-or-later"} readme = "README.md" requires-python = ">=3.9" dynamic = ['version'] [project.urls] Documentation = "https://github.com/lkhq/laniakea-spark" Source = "https://github.com/lkhq/laniakea-spark" [build-system] requires = ["setuptools", "wheel", "pkgconfig"] build-backend = "setuptools.build_meta" [tool.pylint.master] [tool.pylint.format] max-line-length = 100 [tool.pylint."messages control"] disable = [ 'C', 'R', 'fixme', 'unused-argument', 'global-statement', 'logging-format-interpolation', 'attribute-defined-outside-init', 'protected-access', 'broad-except', 'redefined-builtin', ] [tool.pylint.reports] score = 'no' [tool.pylint.typecheck] ignored-modules = [ 'zmq', ] [tool.isort] py_version = 39 profile = "black" multi_line_output = 3 skip_gitignore = true length_sort = true atomic = true [tool.black] target-version = ['py39'] line-length = 100 skip-string-normalization = true laniakea-spark-0.1.1/requirements.txt000066400000000000000000000000731456520135000176610ustar00rootroot00000000000000tomlkit>=0.8 pyzmq>=16 python-debian>=0.1.28 firehose>=0.5 laniakea-spark-0.1.1/setup.cfg000066400000000000000000000002131456520135000162120ustar00rootroot00000000000000[flake8] ignore = E221 max-line-length = 110 extend-ignore = W503, # See https://github.com/PyCQA/pycodestyle/issues/373 E203, laniakea-spark-0.1.1/setup.py000077500000000000000000000016071456520135000161160ustar00rootroot00000000000000#!/usr/bin/env python3 from setuptools import setup from spark import __appname__, __version__ packages = [ 'spark', 'spark.utils', 'spark.runners', ] scripts = { 'console_scripts': [ 'lk-spark = spark.cli:daemon', ], } data_files = [('/var/lib/lkspark/', ['data/ws/README'])] long_description = "" install_requires = ['tomlkit>=0.8', 'pyzmq>=16', 'python-debian>=0.1.28', 'firehose>=0.5'] setup( name=__appname__, version=__version__, scripts=[], packages=packages, data_files=data_files, author="Matthias Klumpp", author_email="matthias@tenstral.net", long_description=long_description, description='Job runner for Laniakea', license="LGPL-3.0+", url="https://laniakea-hq.rtfd.io", python_requires='>=3.9', platforms=['any'], zip_safe=False, entry_points=scripts, install_requires=install_requires, ) laniakea-spark-0.1.1/spark/000077500000000000000000000000001456520135000155155ustar00rootroot00000000000000laniakea-spark-0.1.1/spark/__init__.py000066400000000000000000000015111456520135000176240ustar00rootroot00000000000000# Copyright (C) 2016 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . __appname__ = 'laniakea-spark' __version__ = '0.1.1' laniakea-spark-0.1.1/spark/cli.py000066400000000000000000000026431456520135000166430ustar00rootroot00000000000000# Copyright (C) 2017 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . def daemon(): from argparse import ArgumentParser parser = ArgumentParser(description="Debile build slave") parser.add_argument( "--config", action="store", dest="config", default=None, help="Path to the slave.yaml config file.", ) parser.add_argument( "-s", "--syslog", action="store_true", dest="syslog", help="Log to syslog instead of stderr.", ) parser.add_argument( "-d", "--debug", action="store_true", dest="debug", help="Enable debug messages to stderr." ) from spark.daemon import Daemon d = Daemon() d.run() laniakea-spark-0.1.1/spark/config.py000066400000000000000000000135301456520135000173360ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2017-2021 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . import os import sys import logging as log import platform from typing import List from pathlib import Path import tomlkit class ConfigError(Exception): """Some problem in the configuration was found.""" class LocalConfig: """ Local configuration for the spark daemon. """ CERTS_BASE_DIR = '/etc/laniakea/keys/curve/' def load(self, fname=None): if not fname: fname = '/etc/laniakea/spark.toml' cdata = None with open(fname, encoding='utf8') as toml_file: try: cdata = tomlkit.load(toml_file) except tomlkit.exceptions.ParseError as e: print( 'Unable to parse configuration ({}): {}'.format(fname, str(e)), file=sys.stderr, ) sys.exit(6) self._machine_owner = cdata.get('MachineOwner') self._machine_name = cdata.get('MachineName') if not self._machine_name: self._machine_name = ( Path('/etc/hostname').read_text(encoding='utf-8').strip('\n').strip() ) # read the machine ID self._machine_id = Path('/etc/machine-id').read_text(encoding='utf-8').strip('\n').strip() # make an UUID for this client from the machine name self._make_client_uuid(self._machine_name) self._lighthouse_server = cdata.get('LighthouseServer') if not self._lighthouse_server: raise ConfigError( 'The "LighthouseServer" configuration entry is missing. ' 'Please specify the address of a Lighthouse server.' ) self._max_jobs = int(cdata.get("MaxJobs", 1)) if self._max_jobs < 1: raise ConfigError('The maximum number of jobs can not be < 1.') self._client_cert_fname = os.path.join( self.CERTS_BASE_DIR, 'secret', '{0}-spark_private.sec'.format(self.machine_name) ) self._server_cert_fname = os.path.join( self.CERTS_BASE_DIR, '{0}_lighthouse-server.pub'.format(self.machine_name) ) workspace_root = cdata.get('WorkspaceRoot') if not workspace_root: workspace_root = '/var/lib/lkspark/' self._workspace_dir = os.path.join(workspace_root, 'workspaces') self._job_log_dir = os.path.join(workspace_root, 'logs') self._dput_cf_fname = os.path.join(workspace_root, 'dput.cf') self._architectures = cdata.get("Architectures") if not self._architectures: import re # try to rescue doing some poor mapping to the Debian arch vendor strings # for a couple of common architectures machine_str = platform.machine() if machine_str == 'x86_64': self._architectures = ['amd64'] elif re.match('i?86', machine_str): self._architectures = ['i386'] elif machine_str == 'aarch64': self._architectures = ['arm64'] else: self._architectures = [machine_str] log.warning('Using auto-detected architecture name: {}'.format(machine_str)) self._accepted_job_kinds = cdata.get("AcceptedJobs") if not self._accepted_job_kinds: raise ConfigError( 'The essential "AcceptedJobs" configuration entry is missing - ' 'without accepting any job type, running this daemon is pointless.' ) self._gpg_key_id = cdata.get('GpgKeyID') if not self._gpg_key_id: raise ConfigError('The essential "GpgKeyID" configuration entry is missing.') def _make_client_uuid(self, machine_name): import uuid client_uuid = uuid.uuid5(uuid.UUID('d44a99a2-0b5d-415b-808a-790ad4684309'), machine_name) self._client_uuid = str(client_uuid) @property def machine_id(self) -> str: return self._machine_id @property def client_uuid(self) -> str: return self._client_uuid @property def machine_name(self) -> str: return self._machine_name @property def machine_owner(self) -> str: return self._machine_owner @property def accepted_job_kinds(self) -> List[str]: return self._accepted_job_kinds @property def lighthouse_server(self) -> str: return self._lighthouse_server @property def max_jobs(self) -> int: return self._max_jobs @property def client_cert_fname(self) -> str: return self._client_cert_fname @property def server_cert_fname(self) -> str: return self._server_cert_fname @property def workspace_dir(self) -> str: return self._workspace_dir @property def dput_cf_fname(self) -> str: """Path to our dput.cf filename.""" return self._dput_cf_fname @property def job_log_dir(self) -> str: return self._job_log_dir @property def supported_architectures(self) -> List[str]: return self._architectures @property def gpg_key_id(self) -> str: return self._gpg_key_id laniakea-spark-0.1.1/spark/connection.py000066400000000000000000000214151456520135000202310ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2017-2020 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . import os import json import logging as log from enum import StrEnum import zmq import zmq.auth from spark.utils.misc import to_compact_json class JobStatus(StrEnum): """Returned status ob a Job""" ACCEPTED = 'accepted' # worker accepted the job REJECTED = 'rejected' # worker rejected taking the job SUCCESS = 'success' # success FAILED = 'failed' # job failed class ReplyException(Exception): pass class MessageException(Exception): pass class ServerErrorException(Exception): pass # maximum amount of time to wait for a server response RESPONSE_WAIT_TIME = 15000 # 15sec class ServerConnection: def __init__(self, conf, ctx): if zmq.zmq_version_info() < (4, 0): raise RuntimeError( "Security is not supported in libzmq version < 4.0. libzmq version {0}".format( zmq.zmq_version() ) ) self._conf = conf self._zctx = ctx self._send_attempts = 0 def connect(self): """ Set up an encrypted connection to the Lighthouse server andf run it. """ # construct base data to include in all requests to the server self._base_req = {} self._base_req['machine_name'] = self._conf.machine_name self._base_req['machine_id'] = self._conf.client_uuid # initialize Lighthouse socket self._sock = self._zctx.socket(zmq.REQ) self._sock.setsockopt(zmq.REQ_RELAXED, 1) # set server certificate server_public, _ = zmq.auth.load_certificate(self._conf.server_cert_fname) self._sock.curve_serverkey = server_public # set client certificate client_secret_file = os.path.join(self._conf.client_cert_fname) client_public, client_secret = zmq.auth.load_certificate(client_secret_file) self._sock.curve_secretkey = client_secret self._sock.curve_publickey = client_public # connect self._sock.connect(self._conf.lighthouse_server) self._poller = zmq.Poller() self._poller.register(self._sock, zmq.POLLIN) def reconnect(self): """ Re-establish connection. The lazy answer in case we got no reply from the server for a while. """ self._sock.close() self.connect() def send_job_status(self, job_id, status): req = self.new_base_request() req['request'] = 'job-{}'.format(status) req['uuid'] = job_id try: self._sock.send_string(to_compact_json(req)) except zmq.error.ZMQError as e: self._send_attempt_failed(e) log.error('ZMQ error while sending job status: %s', str(e)) try: sockev = dict(self._poller.poll(RESPONSE_WAIT_TIME)) except zmq.error.ZMQError as e: self._send_attempt_failed() log.error('ZMQ error while waiting for reply: %s', str(e)) if sockev.get(self._sock) == zmq.POLLIN: self._sock.recv_multipart() # discard reply else: log.error('Unable to send job status: No reply from master') def new_base_request(self): """ Get a copy of the base request template. """ return dict(self._base_req) def request_job(self): """ Request a new job from the server. """ # construct job request req = dict(self._base_req) req['request'] = 'job' req['owner'] = self._conf.machine_owner req['accepts'] = self._conf.accepted_job_kinds req['architectures'] = self._conf.supported_architectures # request job self._sock.send_string(to_compact_json(req)) # wait for a reply job_reply_msgs = None try: sev = dict(self._poller.poll(RESPONSE_WAIT_TIME)) except zmq.error.ZMQError as e: self._send_attempt_failed() raise ReplyException('ZMQ error while polling for reply: ' + str(e)) from e if sev.get(self._sock) == zmq.POLLIN: job_reply_msgs = self._sock.recv_multipart() else: self._send_attempt_failed() raise ReplyException('Job request expired (the master server might be unreachable).') if not job_reply_msgs: raise ReplyException('Invalid server response on a job request.') job_reply_raw = job_reply_msgs[0] job_reply = None try: job_reply = json.loads(str(job_reply_raw, 'utf-8')) except Exception as e: raise MessageException( 'Unable to decode server reply ({0}): {1}'.format(job_reply_raw, str(e)) ) from e if not job_reply: log.debug('No new jobs.') return None try: server_error = job_reply.get('error') except Exception as e: raise ServerErrorException( 'Received unexpected server reply: {}'.format(str(job_reply)) ) from e if server_error: raise ServerErrorException( 'Received error message from server: {}'.format(server_error) ) return job_reply def request_archive_info(self): """ Request archive setup information, to know which repositories exist and where to upload to. """ # construct information request req = dict(self._base_req) req['request'] = 'archive-info' # request data self._sock.send_string(to_compact_json(req)) # wait for a reply reply_msgs = None try: sev = dict(self._poller.poll(RESPONSE_WAIT_TIME)) except zmq.error.ZMQError as e: self._send_attempt_failed() raise ReplyException('ZMQ error while polling for setup data reply: ' + str(e)) from e if sev.get(self._sock) == zmq.POLLIN: reply_msgs = self._sock.recv_multipart() else: self._send_attempt_failed() raise ReplyException( 'Request for archive data expired (the master server might be unreachable).' ) if not reply_msgs: raise ReplyException('Invalid server response on a job request.') reply_raw = reply_msgs[0] reply_data = None try: reply_data = json.loads(str(reply_raw, 'utf-8')) except Exception as e: raise MessageException( 'Unable to decode server reply ({0}): {1}'.format(reply_raw, str(e)) ) from e if not reply_data: log.debug('No archive data configured.') return None try: server_error = reply_data.get('error') except Exception as e: raise ServerErrorException( 'Received unexpected server reply: {}'.format(str(reply_data)) ) from e if server_error: raise ServerErrorException( 'Received error message from server: {}'.format(server_error) ) return reply_data def _send_attempt_failed(self, error=None): self._send_attempts = self._send_attempts + 1 if self._send_attempts >= 6: if error: log.error('Send attempts expired ({}), reconnecting...'.format(str(error))) else: log.error('Send attempts expired, reconnecting...') self.reconnect() self._send_attempts = 0 def send_str_noreply(self, s): if type(s) is str: data = s.encode('utf-8') elif type(s) is not bytes: data = str(s).encode('utf-8') self._sock.send(data, copy=False, track=True) try: sev = dict(self._poller.poll(RESPONSE_WAIT_TIME)) except zmq.error.ZMQError as e: self._send_attempt_failed(e) if sev.get(self._sock) == zmq.POLLIN: self._sock.recv_multipart() # discard reply else: self._send_attempt_failed() log.info('Received no ACK from server for noreply request.') laniakea-spark-0.1.1/spark/daemon.py000066400000000000000000000057521456520135000173430ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2016-2020 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . import sys import shutil import logging as log from multiprocessing import Process import zmq from spark.config import LocalConfig from spark.worker import Worker from spark.connection import ServerConnection class Daemon: def __init__(self, log_level=None): if not log_level: log_level = log.INFO log.basicConfig(level=log_level, format="[%(levelname)s] %(message)s") def run_worker_process(self, worker_name: str, is_primary: bool): """ Set up connection for a new worker process and launch it. This function is executed in a new process. """ zctx = zmq.Context() # initialize Lighthouse connection conn = ServerConnection(self._conf, zctx) # connect conn.connect() log.info( 'Running {0} on {1} ({2})'.format( worker_name, self._conf.machine_name, self._conf.client_uuid ) ) w = Worker(self._conf, conn, is_primary=is_primary) w.run() def run(self): # check Python platform version - 3.5 works while 3.6 or higher is properly tested pyversion = sys.version_info if pyversion < (3, 11): raise RuntimeError( 'Laniakea-Spark needs Python >= 3.11 to work. Please upgrade your Python version.' ) if not shutil.which('debspawn'): log.warning( 'The "debspawn" tool was not found in PATH, we will not be able to run most actions.' ) self._conf = LocalConfig() self._conf.load() log.info('Maximum number of parallel jobs: {0}'.format(self._conf.max_jobs)) # initialize workers if self._conf.max_jobs == 1: # don't use multiprocess when our maximum amount of jobs is just 1 self.run_worker_process('worker_0', is_primary=True) else: is_primary = True for i in range(0, self._conf.max_jobs): worker_name = 'worker_{}'.format(i) p = Process(target=self.run_worker_process, args=(worker_name, is_primary)) p.name = worker_name p.start() is_primary = False laniakea-spark-0.1.1/spark/joblog.py000066400000000000000000000053771456520135000173570ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2017-2020 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . import threading from io import StringIO from contextlib import contextmanager from spark.utils.misc import to_compact_json class JobLog: """ Send status information (usually in form of stdout/stderr output) for a specific job to the server as well as to the local config file. """ def __init__(self, lhconn, job_id, log_fname): self._conn = lhconn self._buf = StringIO() self._file = open(log_fname, 'w', encoding='utf8') self._last_msg_excerpt = '' self._job_id = job_id self._msg_template = self._conn.new_base_request() self._msg_template['request'] = 'job-status' self._msg_template['uuid'] = str(job_id) self._have_output = False self._closed = False self._send_timed() # start timer def write(self, s): if isinstance(s, (bytes, bytearray)): s = str(s, 'utf-8') self._buf.write(s) self._file.write(s) self._have_output = True def flush(self): self._file.flush() def _send_timed(self): if self._have_output: self._send_buffer() if not self._closed: threading.Timer(30.0, self._send_timed).start() def _send_buffer(self): if not self._have_output: return self._have_output = False log_excerpt = self._buf.getvalue() self._buf = StringIO() req = dict(self._msg_template) # copy the template req['log_excerpt'] = log_excerpt self._conn.send_str_noreply(to_compact_json(req)) self._last_msg_excerpt = log_excerpt def close(self): if self._have_output: self._send_buffer() self._closed = True self._file.close() @property def job_id(self) -> str: return self._job_id @contextmanager def job_log(lhconn, job_id, log_fname): jlog = JobLog(lhconn, job_id, log_fname) try: yield jlog finally: jlog.close() laniakea-spark-0.1.1/spark/runners/000077500000000000000000000000001456520135000172115ustar00rootroot00000000000000laniakea-spark-0.1.1/spark/runners/__init__.py000066400000000000000000000021611456520135000213220ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (C) 2017-2020 Matthias Klumpp # # Licensed under the GNU Lesser General Public License Version 3 # # This program is free software: you can redistribute it and/or modify # it under the terms of the GNU Lesser General Public License as published by # the Free Software Foundation, either version 3 of the license, or # (at your option) any later version. # # This software is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU Lesser General Public License for more details. # # You should have received a copy of the GNU Lesser General Public License # along with this software. If not, see . import importlib # Determine which runner is responsible for which job type. PLUGINS = { 'package-build': 'spark.runners.debspawn', 'os-image-build': 'spark.runners.image_build', } def load_module(what): path = PLUGINS[what] mod = importlib.import_module(path) return (mod.run, mod.get_version) laniakea-spark-0.1.1/spark/runners/debspawn.py000066400000000000000000000135331456520135000213730ustar00rootroot00000000000000# -*- coding: utf-8 -*- # # Copyright (c) 2012-2013 Paul Tagliamonte # Copyright (c) 2016-2018 Matthias Klumpp # # Permission is hereby granted, free of charge, to any person obtaining a # copy of this software and associated documentation files (the "Software"), # to deal in the Software without restriction, including without limitation # the rights to use, copy, modify, merge, publish, distribute, sublicense, # and/or sell copies of the Software, and to permit persons to whom the # Software is furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL # THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER # DEALINGS IN THE SOFTWARE. import os import re import glob from io import StringIO from datetime import timedelta import firehose.parsers.gcc as fgcc from firehose.model import Stats, Analysis from spark.utils import RunnerError, RunnerResult from spark.utils.command import safe_run, run_logged, run_command from spark.utils.firehose import create_firehose STATS = re.compile('Build needed (?P