pax_global_header00006660000000000000000000000064143436063560014524gustar00rootroot0000000000000052 comment=f1e1db4d8a2863fc9fe7914c4df4a17085999b29 pyaml_env-1.2.1/000077500000000000000000000000001434360635600135175ustar00rootroot00000000000000pyaml_env-1.2.1/.github/000077500000000000000000000000001434360635600150575ustar00rootroot00000000000000pyaml_env-1.2.1/.github/ISSUE_TEMPLATE/000077500000000000000000000000001434360635600172425ustar00rootroot00000000000000pyaml_env-1.2.1/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000007211434360635600217340ustar00rootroot00000000000000--- name: Bug report about: Create a report to help us improve title: "[BUG]" labels: bug assignees: mkaranasou --- **Describe the bug** A clear and concise description of what the bug is. **To Reproduce** Steps to reproduce the behavior: 1. Provide a YAML example 2. Run 3. Results **Expected behavior** A clear and concise description of what you expected to happen. **Additional context** Add any other context about the problem here. - Python version - OS pyaml_env-1.2.1/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000011551434360635600227710ustar00rootroot00000000000000--- name: Feature request about: Suggest an idea for this project title: "[FEATURE]" labels: enhancement assignees: mkaranasou --- **Is your feature request related to a problem? Please describe.** A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. **Additional context** Add any other context or screenshots about the feature request here. pyaml_env-1.2.1/.github/workflows/000077500000000000000000000000001434360635600171145ustar00rootroot00000000000000pyaml_env-1.2.1/.github/workflows/codeql-analysis.yml000066400000000000000000000046141434360635600227340ustar00rootroot00000000000000# For most projects, this workflow file will not need changing; you simply need # to commit it to your repository. # # You may wish to alter this file to override the set of languages analyzed, # or to provide custom queries or build logic. # # ******** NOTE ******** # We have attempted to detect the languages in your repository. Please check # the `language` matrix defined below to confirm you have the correct set of # supported CodeQL languages. # name: "CodeQL" on: push: branches: [ main ] pull_request: # The branches below must be a subset of the branches above branches: [ main ] schedule: - cron: '17 20 * * 5' jobs: analyze: name: Analyze runs-on: ubuntu-latest permissions: actions: read contents: read security-events: write strategy: fail-fast: false matrix: language: [ 'python' ] # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ] # Learn more: # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed steps: - name: Checkout repository uses: actions/checkout@v2 # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v1 with: languages: ${{ matrix.language }} # If you wish to specify custom queries, you can do so here or in a config file. # By default, queries listed here will override any specified in a config file. # Prefix the list here with "+" to use these queries and those in the config file. # queries: ./path/to/local/query, your-org/your-repo/queries@main # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). # If this step fails, then you should remove it and run the build manually (see below) - name: Autobuild uses: github/codeql-action/autobuild@v1 # ℹ️ Command-line programs to run using the OS shell. # 📚 https://git.io/JvXDl # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines # and modify them (or add more) to build your code if your project # uses a compiled language #- run: | # make bootstrap # make release - name: Perform CodeQL Analysis uses: github/codeql-action/analyze@v1 pyaml_env-1.2.1/.github/workflows/python-app.yml000066400000000000000000000022271434360635600217410ustar00rootroot00000000000000# This workflow will install Python dependencies, run tests and lint with a single version of Python # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions name: Python application on: push: branches: [ main ] pull_request: branches: [ main ] jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python 3.9 uses: actions/setup-python@v2 with: python-version: 3.9 - name: Install dependencies run: | python -m pip install --upgrade pip pip install flake8 pytest pip install . # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - name: Lint with flake8 run: | # stop the build if there are Python syntax errors or undefined names flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest run: | pytest pyaml_env-1.2.1/.github/workflows/python-publish.yml000066400000000000000000000015401434360635600226240ustar00rootroot00000000000000# This workflow will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries name: Upload Python Package on: release: types: [created] jobs: deploy: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: '3.x' - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools wheel twine - name: Build and publish env: TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} run: | python setup.py sdist bdist_wheel twine upload dist/* pyaml_env-1.2.1/.gitignore000066400000000000000000000034171434360635600155140ustar00rootroot00000000000000# Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller # Usually these files are written by a python script from a template # before PyInstaller builds the exe, so as to inject date/other infos into it. *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. # However, in case of collaboration, if having platform-specific dependencies or dependencies # having no cross-platform support, pipenv may install dependencies that don't work, or not # install all needed dependencies. #Pipfile.lock # PEP 582; used by e.g. github.com/David-OConnor/pyflow __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ .idea/ pyaml_env-1.2.1/LICENSE000066400000000000000000000020601434360635600145220ustar00rootroot00000000000000MIT License Copyright (c) 2021 Maria Karanasou Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. pyaml_env-1.2.1/MANIFEST.in000066400000000000000000000000311434360635600152470ustar00rootroot00000000000000include requirements.txt pyaml_env-1.2.1/README.md000066400000000000000000000265371434360635600150130ustar00rootroot00000000000000[![Downloads](https://static.pepy.tech/personalized-badge/pyaml-env?period=total&units=none&left_color=black&right_color=green&left_text=Downloads)](https://pepy.tech/project/pyaml-env) [![Tests and linting](https://github.com/mkaranasou/pyaml_env/actions/workflows/python-app.yml/badge.svg)](https://github.com/mkaranasou/pyaml_env/actions/workflows/python-app.yml) [![CodeQL](https://github.com/mkaranasou/pyaml_env/actions/workflows/codeql-analysis.yml/badge.svg)](https://github.com/mkaranasou/pyaml_env/actions/workflows/codeql-analysis.yml) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![Upload Python Package](https://github.com/mkaranasou/pyaml_env/actions/workflows/python-publish.yml/badge.svg)](https://github.com/mkaranasou/pyaml_env/actions/workflows/python-publish.yml) # Python YAML configuration with environment variables parsing ## TL;DR A very small library that parses a yaml configuration file and it resolves the environment variables, so that no secrets are kept in text. ### Install ```bash pip install pyaml-env ``` ### How to use: --- #### Basic Usage: Environment variable parsing This yaml file: ```yaml database: name: test_db username: !ENV ${DB_USER} password: !ENV ${DB_PASS} url: !ENV 'http://${DB_BASE_URL}:${DB_PORT}' ``` given that we've set these: ```bash export $DB_USER=super_secret_user export $DB_PASS=extra_super_secret_password export $DB_BASE_URL=localhost export $DB_PORT=5432 ``` becomes this: ```python from pyaml_env import parse_config config = parse_config('path/to/config.yaml') print(config) # outputs the following, with the environment variables resolved { 'database': { 'name': 'test_db', 'username': 'super_secret_user', 'password': 'extra_super_secret_password', 'url': 'http://localhost:5432', } } ``` --- #### Attribute Access using `BaseConfig` Which can also become this: ```python from pyaml_env import parse_config, BaseConfig config = BaseConfig(parse_config('path/to/config.yaml')) # you can then access the config properties as atrributes # I'll explain why this might be useful in a bit. print(config.database.url) ``` --- #### Default Values with `:` You can also set default values for when the environment variables are not set for some reason, using the `default_sep` kwarg (**which is `:` by default**) like this: ```yaml databse: name: test_db username: !ENV ${DB_USER:paws} password: !ENV ${DB_PASS:meaw2} url: !ENV 'http://${DB_BASE_URL:straight_to_production}:${DB_PORT}' ``` And if no environment variables are found then we get: ```python from pyaml_env import parse_config config = parse_config('path/to/config.yaml') print(config) { 'database': { 'name': 'test_db', 'username': 'paws', 'password': 'meaw2', 'url': 'http://straight_to_production:N/A', } } ``` **NOTE 0**: Special characters like `*`, `{` etc. are not currently supported as separators. Let me know if you'd like them handled also. **NOTE 1**: If you set `tag` to `None`, then, the current behavior is that environment variables in all places in the yaml will be resolved (if set). --- #### Datatype parsing with yaml's tag:yaml.org,2002: ```python # because this is not allowed: # data1: !TAG !!float ${ENV_TAG2:27017} # use tag:yaml.org,2002:datatype to convert value: test_data = ''' data0: !TAG ${ENV_TAG1} data1: !TAG tag:yaml.org,2002:float ${ENV_TAG2:27017} data2: !!float 1024 data3: !TAG ${ENV_TAG2:some_value} data4: !TAG tag:yaml.org,2002:bool ${ENV_TAG2:false} ''' ``` Will become: ```python os.environ['ENV_TAG1'] = "1024" config = parse_config(data=test_data, tag='!TAG') print(config) { 'data0': '1024', 'data1': 27017.0, 'data2': 1024.0, 'data3': 'some_value', 'data4': False } ``` [reference in yaml code](https://github.com/yaml/pyyaml/blob/master/lib/yaml/parser.py#L78) --- #### If nothing matches: `N/A` as `default_value`: If no defaults are found and no environment variables, the `default_value` (**which is `N/A` by default**) is used: ```python { 'database': { 'name': 'test_db', 'username': 'N/A', 'password': 'N/A', 'url': 'http://N/A:N/A', } } ``` Which, of course, means something went wrong and we need to set the correct environment variables. If you want this process to fail if a *default value* is not found, you can set the `raise_if_na` flag to `True`. For example: ```yaml test1: data0: !TEST ${ENV_TAG1:has_default}/somethingelse/${ENV_TAG2:also_has_default} data1: !TEST ${ENV_TAG2} ``` will raise a `ValueError` because `data1: !TEST ${ENV_TAG2}` there is no default value for `ENV_TAG2` in this line. --- #### Using a different loader: The default yaml loader is `yaml.SafeLoader`. If you need to work with serialized Python objects, you can specify a different loader. So given a class: ```python class OtherLoadTest: def __init__(self): self.data0 = 'it works!' self.data1 = 'this works too!' ``` Which has become a yaml output like the following using `yaml.dump(OtherLoadTest())`: ```yaml !!python/object:__main__.OtherLoadTest data0: it works! data1: this works too! ``` You can use `parse_config` to load the object like this: ```python import yaml from pyaml_env import parse_config other_load_test = parse_config(path='path/to/config.yaml', loader=yaml.UnsafeLoader) print(other_load_test) <__main__.OtherLoadTest object at 0x7fc38ccd5470> ``` --- ## Long story: Load a YAML configuration file and resolve any environment variables ![](https://cdn-images-1.medium.com/max/11700/1*4s_GrxE5sn2p2PNd8fS-6A.jpeg) If you’ve worked with Python projects, you’ve probably have stumbled across the many ways to provide configuration. I am not going to go through all the ways here, but a few of them are: * using .ini files * using a python class * using .env files * using JSON or XML files * using a yaml file And so on. I’ve put some useful links about the different ways below, in case you are interested in digging deeper. My preference is working with yaml configuration because I usually find very handy and easy to use and I really like that yaml files are also used in e.g. docker-compose configuration so it is something most are familiar with. For yaml parsing I use the [PyYAML](https://pyyaml.org/wiki/PyYAMLDocumentation) Python library. In this article we’ll talk about the yaml file case and more specifically what you can do to **avoid keeping your secrets, e.g. passwords, hosts, usernames etc, directly on it**. Let’s say we have a very simple example of a yaml file configuration: database: name: database_name user: me password: very_secret_and_complex host: localhost port: 5432 ws: user: username password: very_secret_and_complex_too host: localhost When you come to a point where you need to deploy your project, it is not really safe to have passwords and sensitive data in a plain text configuration file lying around on your production server. That’s where [**environment variables](https://medium.com/dataseries/hiding-secret-info-in-python-using-environment-variables-a2bab182eea) **come in handy. So the goal here is to be able to easily replace the very_secret_and_complex password with input from an environment variable, e.g. DB_PASS, so that this variable only exists when you set it and run your program instead of it being hardcoded somewhere. For PyYAML to be able to resolve environment variables, we need three main things: * A regex pattern for the environment variable identification e.g. pattern = re.compile(‘.*?\${(\w+)}.*?’) * A tag that will signify that there’s an environment variable (or more) to be parsed, e.g. !ENV. * And a function that the loader will use to resolve the environment variables ```python def constructor_env_variables(loader, node): """ Extracts the environment variable from the node's value :param yaml.Loader loader: the yaml loader :param node: the current node in the yaml :return: the parsed string that contains the value of the environment variable """ value = loader.construct_scalar(node) match = pattern.findall(value) if match: full_value = value for g in match: full_value = full_value.replace( f'${{{g}}}', os.environ.get(g, g) ) return full_value return value ``` Example of a YAML configuration with environment variables: database: name: database_name user: !ENV ${DB_USER} password: !ENV ${DB_PASS} host: !ENV ${DB_HOST} port: 5432 ws: user: !ENV ${WS_USER} password: !ENV ${WS_PASS} host: !ENV ‘[https://${CURR_ENV}.ws.com.local'](https://${CURR_ENV}.ws.com.local') This can also work **with more than one environment variables** declared in the same line for the same configuration parameter like this: ws: user: !ENV ${WS_USER} password: !ENV ${WS_PASS} host: !ENV '[https://${CURR_ENV}.ws.com.](https://${CURR_ENV}.ws.com.local')[${MODE}](https://${CURR_ENV}.ws.com.local')' # multiple env var And how to use this: First set the environment variables. For example, for the DB_PASS : export DB_PASS=very_secret_and_complex Or even better, so that the password is not echoed in the terminal: read -s ‘Database password: ‘ db_pass export DB_PASS=$db_pass ```python # To run this: # export DB_PASS=very_secret_and_complex # python use_env_variables_in_config_example.py -c /path/to/yaml # do stuff with conf, e.g. access the database password like this: conf['database']['DB_PASS'] if __name__ == '__main__': parser = argparse.ArgumentParser(description='My awesome script') parser.add_argument( "-c", "--conf", action="store", dest="conf_file", help="Path to config file" ) args = parser.parse_args() conf = parse_config(path=args.conf_file) ``` Then you can run the above script: ```bash python use_env_variables_in_config_example.py -c /path/to/yaml ``` And in your code, do stuff with conf, e.g. access the database password like this: `conf['database']['DB_PASS']` I hope this was helpful. Any thoughts, questions, corrections and suggestions are very welcome :) ## Useful links [**The Many Faces and Files of Python Configs** *As we cling harder and harder to Dockerfiles, Kubernetes, or any modern preconfigured app environment, our dependency…*hackersandslackers.com](https://hackersandslackers.com/simplify-your-python-projects-configuration/) [**4 Ways to manage the configuration in Python** *I’m not a native speaker. Sorry for my english. Please understand.*hackernoon.com](https://hackernoon.com/4-ways-to-manage-the-configuration-in-python-4623049e841b) [**Python configuration files** *A common need when writing an application is loading and saving configuration values in a human-readable text format…*www.devdungeon.com](https://www.devdungeon.com/content/python-configuration-files) [**Configuration files in Python** *Most interesting programs need some kind of configuration: Content Management Systems like WordPress blogs, WikiMedia…*martin-thoma.com](https://martin-thoma.com/configuration-files-in-python/) Buy Me A Coffee pyaml_env-1.2.1/_config.yml000066400000000000000000000000331434360635600156420ustar00rootroot00000000000000theme: jekyll-theme-minimalpyaml_env-1.2.1/requirements.txt000066400000000000000000000000231434360635600167760ustar00rootroot00000000000000PyYAML>=5.0, <=7.0 pyaml_env-1.2.1/setup.cfg000066400000000000000000000011751434360635600153440ustar00rootroot00000000000000[metadata] name = pyaml_env version = 1.2.1 author = Maria Karanasou author_email = karanasou@gmail.com description = Provides yaml file parsing with environment variable resolution long_description = file: README.md long_description_content_type = text/markdown url = https://github.com/mkaranasou/pyaml_env project_urls = Bug Tracker = https://github.com/mkaranasou/pyaml_env/issues classifiers = Programming Language :: Python :: 3 License :: OSI Approved :: MIT License Operating System :: OS Independent [options] package_dir = = src packages = find: python_requires = >=3.6 [options.packages.find] where = srcpyaml_env-1.2.1/setup.py000066400000000000000000000016251434360635600152350ustar00rootroot00000000000000from __future__ import absolute_import from setuptools import setup # To use a consistent encoding from codecs import open from os import path here = path.abspath(path.dirname(__file__)) # README as the long description with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() REQUIREMENTS = [i.strip() for i in open('requirements.txt').readlines()] tests_require = [ 'pytest', ] setup(name='pyaml_env', version='1.2.1', description='Provides yaml file parsing with ' 'environment variable resolution', long_description=long_description, tests_require=tests_require, extras_require={ 'test': tests_require, }, test_suite='pytest.collector', install_requires=REQUIREMENTS, include_package_data=True, package_dir={'': 'src'}, packages=[ 'pyaml_env', ], ) pyaml_env-1.2.1/src/000077500000000000000000000000001434360635600143065ustar00rootroot00000000000000pyaml_env-1.2.1/src/pyaml_env/000077500000000000000000000000001434360635600163005ustar00rootroot00000000000000pyaml_env-1.2.1/src/pyaml_env/__init__.py000066400000000000000000000001641434360635600204120ustar00rootroot00000000000000from .parse_config import parse_config from .base_config import BaseConfig __all__ = ['parse_config', 'BaseConfig']pyaml_env-1.2.1/src/pyaml_env/base_config.py000066400000000000000000000016511434360635600211140ustar00rootroot00000000000000from typing import Any class BaseConfig: """ A base Config class to get """ def __init__(self, config_dict): if config_dict: self.__dict__.update(**{ k: v for k, v in self.__class__.__dict__.items() if '__' not in k and not callable(v) }) self.__dict__.update(**config_dict) self._is_validated = False self._is_valid = False self._errors = [] self.__dict__ = self.__handle_inner_structures() def __handle_inner_structures(self): for k, v in self.__dict__.items(): if isinstance(v, dict): self.__dict__[k] = BaseConfig(v) return self.__dict__ def __getattr__(self, field_name: str) -> Any: return self.__dict__[field_name] @property def errors(self): return self._errors def validate(self): raise NotImplementedError() pyaml_env-1.2.1/src/pyaml_env/parse_config.py000066400000000000000000000111131434360635600213060ustar00rootroot00000000000000import os import re import yaml def parse_config( path=None, data=None, tag='!ENV', default_sep=':', default_value='N/A', raise_if_na=False, loader=yaml.SafeLoader, encoding='utf-8' ): """ Load yaml configuration from path or from the contents of a file (data) and resolve any environment variables. The environment variables must have the tag e.g. !ENV *before* them and be in this format to be parsed: ${VAR_NAME} E.g.: databse: name: test_db username: !ENV ${DB_USER:paws} password: !ENV ${DB_PASS:meaw2} url: !ENV 'http://${DB_BASE_URL:straight_to_production}:${DB_PORT:12345}' :param str path: the path to the yaml file :param str data: the yaml data itself as a stream :param str tag: the tag to look for, if None, all env variables will be resolved. :param str default_sep: if any default values are set, use this field to separate them from the enironment variable name. E.g. ':' can be used. :param str default_value: the tag to look for :param bool raise_if_na: raise an exception if there is no default value set for the env variable. :param Type[yaml.loader] loader: Specify which loader to use. Defaults to yaml.SafeLoader :param str encoding: the encoding of the data if a path is specified, defaults to utf-8 :return: the dict configuration :rtype: dict[str, T] """ default_sep = default_sep or '' default_value = default_value or '' default_sep_pattern = r'(' + default_sep + '[^}]+)?' if default_sep else '' pattern = re.compile( r'.*?\$\{([^}{' + default_sep + r']+)' + default_sep_pattern + r'\}.*?') loader = loader or yaml.SafeLoader # the tag will be used to mark where to start searching for the pattern # e.g. a_key: !ENV somestring${ENV_VAR}other_stuff_follows loader.add_implicit_resolver(tag, pattern, first=[tag]) # For inner type conversions because double tags do not work, e.g. !ENV !!float type_tag = 'tag:yaml.org,2002:' type_tag_pattern = re.compile(f'({type_tag}\w+\s)') def constructor_env_variables(loader, node): """ Extracts the environment variable from the yaml node's value :param yaml.Loader loader: the yaml loader (as defined above) :param node: the current node (key-value) in the yaml :return: the parsed string that contains the value of the environment variable or the default value if defined for the variable. If no value for the variable can be found, then the value is replaced by default_value='N/A' """ value = loader.construct_scalar(node) match = pattern.findall(value) # to find all env variables in line dt = ''.join(type_tag_pattern.findall(value)) or '' value = value.replace(dt, '') if match: full_value = value for g in match: curr_default_value = default_value env_var_name = g env_var_name_with_default = g if default_sep and isinstance(g, tuple) and len(g) > 1: env_var_name = g[0] env_var_name_with_default = ''.join(g) found = False for each in g: if default_sep in each: _, curr_default_value = each.split(default_sep, 1) found = True break if not found and raise_if_na: raise ValueError( f'Could not find default value for {env_var_name}' ) full_value = full_value.replace( f'${{{env_var_name_with_default}}}', os.environ.get(env_var_name, curr_default_value) ) if dt: # do one more roundtrip with the dt constructor: node.value = full_value node.tag = dt.strip() return loader.yaml_constructors[node.tag](loader, node) return full_value return value loader.add_constructor(tag, constructor_env_variables) if path: with open(path, encoding=encoding) as conf_data: return yaml.load(conf_data, Loader=loader) elif data: return yaml.load(data, Loader=loader) else: raise ValueError('Either a path or data should be defined as input') pyaml_env-1.2.1/tests/000077500000000000000000000000001434360635600146615ustar00rootroot00000000000000pyaml_env-1.2.1/tests/__init__.py000066400000000000000000000000701434360635600167670ustar00rootroot00000000000000import os import sys sys.path.insert(0, (os.getcwd())) pyaml_env-1.2.1/tests/pyaml_env_tests/000077500000000000000000000000001434360635600200755ustar00rootroot00000000000000pyaml_env-1.2.1/tests/pyaml_env_tests/__init__.py000066400000000000000000000000001434360635600221740ustar00rootroot00000000000000pyaml_env-1.2.1/tests/pyaml_env_tests/test_base_config.py000066400000000000000000000043611434360635600237510ustar00rootroot00000000000000import os import unittest from pyaml_env import BaseConfig class TestBaseConfig(unittest.TestCase): def setUp(self): self.simple_data = { 'a': 1, 'b': 2, } self.complex_data = { 'a': { 'b': { 'c': [1, 2], 'd': { 'e': 12, 'f': 'test' } } }, 'g': { 'h': { 'i': 'ai', 'j': 'jay' }, 'k': [1, 3, 5] } } def test_base_config_simple_structure(self): base_config = BaseConfig(self.simple_data) self.assertTrue( hasattr(base_config, 'a') ) self.assertTrue( hasattr(base_config, 'b') ) print(base_config.a) print(base_config.b) self.assertEqual(self.simple_data['a'], base_config.a) self.assertEqual(self.simple_data['b'], base_config.b) def test_base_config_complex_structure(self): base_config = BaseConfig(self.complex_data) self.assertTrue( hasattr(base_config, 'a') ) self.assertTrue( hasattr(base_config, 'g') ) print(base_config.a) print(base_config.g) print(base_config.g.h) print(base_config.g.h.i) print(base_config.g.h.j) print(base_config.g.k) print(base_config.a.b) print(base_config.a.b.c) print(base_config.a.b.d) print(base_config.a.b.d.e) print(base_config.a.b.d.f) self.assertIsInstance(base_config.a, BaseConfig) self.assertIsInstance(base_config.g, BaseConfig) self.assertIsInstance(base_config.g.h, BaseConfig) self.assertIsInstance(base_config.g.h.i, str) self.assertIsInstance(base_config.g.h.j, str) self.assertIsInstance(base_config.g.k, list) self.assertIsInstance(base_config.a.b, BaseConfig) self.assertIsInstance(base_config.a.b.c, list) self.assertIsInstance(base_config.a.b.d, BaseConfig) self.assertIsInstance(base_config.a.b.d.e, int) self.assertIsInstance(base_config.a.b.d.f, str)pyaml_env-1.2.1/tests/pyaml_env_tests/test_parse_config.py000066400000000000000000000557441434360635600241640ustar00rootroot00000000000000import os import unittest import yaml from yaml.constructor import ConstructorError from pyaml_env import parse_config class UnsafeLoadTest: def __init__(self): self.data0 = 'it works!' self.data1 = 'this works too!' class TestParseConfig(unittest.TestCase): def setUp(self): self.test_file_name = f'{os.path.abspath(".")}/testfile.yaml' self.env_var1 = 'ENV_TAG1' self.env_var2 = 'ENV_TAG2' self.env_var3 = 'ENV_TAG3' self.env_var4 = 'ENV*)__TAG101sfdarg' os.unsetenv(self.env_var1) os.unsetenv(self.env_var2) os.unsetenv(self.env_var3) os.unsetenv(self.env_var4) def tearDown(self): if self.env_var1 in os.environ: del os.environ[self.env_var1] if self.env_var2 in os.environ: del os.environ[self.env_var2] if self.env_var3 in os.environ: del os.environ[self.env_var3] if self.env_var4 in os.environ: del os.environ[self.env_var4] if os.path.isfile(self.test_file_name): os.remove(self.test_file_name) def test_parse_config_data(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !ENV ${ENV_TAG1} data1: !ENV ${ENV_TAG2} ''' config = parse_config(data=test_data) expected_config = { 'test1': { 'data0': 'it works!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_file_path(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !ENV ${ENV_TAG1} data1: !ENV ${ENV_TAG2} ''' with open(self.test_file_name, 'w') as test_file: test_file.write(test_data) config = parse_config(path=self.test_file_name) expected_config = { 'test1': { 'data0': 'it works!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_diff_tag(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1} data1: !TEST ${ENV_TAG2} ''' config = parse_config(data=test_data, tag='!TEST') expected_config = { 'test1': { 'data0': 'it works!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_diff_tag_file_path(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1} data1: !TEST ${ENV_TAG2} ''' with open(self.test_file_name, 'w') as test_file: test_file.write(test_data) config = parse_config(path=self.test_file_name, tag='!TEST') expected_config = { 'test1': { 'data0': 'it works!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_more_than_one_env_value(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2} ''' config = parse_config(data=test_data, tag='!TEST') expected_config = { 'test1': { 'data0': 'it works!/somethingelse/this works too!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_more_than_one_env_value_file_path(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2} ''' with open(self.test_file_name, 'w') as test_file: test_file.write(test_data) config = parse_config(path=self.test_file_name, tag='!TEST') expected_config = { 'test1': { 'data0': 'it works!/somethingelse/this works too!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_parse_config_no_default_separator(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:default2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=None) expected_config = { 'test1': { 'data0': 'N/A/somethingelse/N/A', 'data1': 'N/A' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_special_char(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var4] = 'this works too!' # default separator exists in the env_var4 name, which should yield a # misread in the environment variable name, which leads to sre_constants.error test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG4} data1: !TEST ${ENV_TAG4:default2} ''' import sre_constants with self.assertRaises(sre_constants.error): _ = parse_config(data=test_data, tag='!TEST', default_sep='*') config = parse_config(data=test_data, tag='!TEST', default_sep='\*') expected_config = { 'test1': { 'data0': 'N/A/somethingelse/N/A', 'data1': 'N/A' } } self.assertDictEqual(config, expected_config) def test_parse_config_default_separator_special_char_not_resolved(self): os.environ[self.env_var4] = 'this works too!' # default separator exists in the env_var4 name, which should yield a # misread in the environment variable name, which leads to sre_constants.error test_data = ''' test1: data0: !TEST ${ENV_TAG1*default1}/somethingelse/${ENV_TAG4} data1: !TEST ${ENV_TAG4:default2} ''' import sre_constants with self.assertRaises(sre_constants.error): _ = parse_config(data=test_data, tag='!TEST', default_sep='*') config = parse_config(data=test_data, tag='!TEST', default_sep='\*') expected_config = { 'test1': { 'data0': 'N/A/somethingelse/N/A', 'data1': 'N/A' } } self.assertDictEqual(config, expected_config) def test_parse_config_default_value_arg(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:default2} ''' config = parse_config(data=test_data, tag='!TEST', default_value='++') expected_config = { 'test1': { 'data0': 'default1/somethingelse/++', 'data1': 'default2' } } self.assertDictEqual( config, expected_config ) def test_parse_config_diff_default_separator(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1!default1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2!default2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep='!') expected_config = { 'test1': { 'data0': 'default1/somethingelse/N/A', 'data1': 'default2' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator(self): # os.environ[self.env_var1] = 'it works!' # os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:default2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'default1/somethingelse/N/A', 'data1': 'default2' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_two_env_vars_in_one_line(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'default1/somethingelse/default2', 'data1': 'N/A' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_two_env_vars_in_one_line_extra_chars(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:defaul^{t1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'defaul^{t1/somethingelse/default2', 'data1': 'N/A' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_illegal_char_default_value(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:defaul^{}t1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'defaul^{t1}/somethingelse/default2', 'data1': 'N/A' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_diff_default_value(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2} ''' config = parse_config( data=test_data, tag='!TEST', default_sep=':', default_value='DEFAULT_VALUE' ) expected_config = { 'test1': { 'data0': 'default1/somethingelse/default2', 'data1': 'DEFAULT_VALUE' } } self.assertDictEqual( config, expected_config ) def test_parse_config_raise_if_na(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2} ''' with self.assertRaises(ValueError): _ = parse_config( data=test_data, tag='!TEST', default_sep=':', raise_if_na=True ) def test_parse_config_raise_if_na_not_raised(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:default1}/somethingelse/${ENV_TAG2:default2} data1: !TEST ${ENV_TAG2:default3} ''' expected_config = { 'test1': { 'data0': 'default1/somethingelse/default2', 'data1': 'default3' } } # no na so this should not raise anything config = parse_config( data=test_data, tag='!TEST', default_sep=':', raise_if_na=True ) self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_strong_password(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:NoHtnnmEuluGp2boPvGQkGrXqTAtBvIVz9VRmV65}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'NoHtnnmEuluGp2boPvGQkGrXqTAtBvIVz9VRmV65/somethingelse/N/A', 'data1': '0.0.0.0' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_var_chars(self): test_data = ''' test1: data0: !TEST ${ENV_TAG1:35xV*+/\gPEFGxrg}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': '35xV*+/\gPEFGxrg/somethingelse/N/A', 'data1': '0.0.0.0' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_var_chars_env_var(self): os.environ[self.env_var1] = 'test' test_data = ''' test1: data0: !TEST ${ENV_TAG1:35xV*+/\gPEFGxrg}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'test/somethingelse/N/A', 'data1': '0.0.0.0' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_strong_password_overwritten_by_env_var(self): os.environ[self.env_var1] = "myWeakPassword" test_data = ''' test1: data0: !TEST ${ENV_TAG1:NoHtnnmEuluGp2boPvGQkGrXqTAtBvIVz9VRmV65}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': 'myWeakPassword/somethingelse/N/A', 'data1': '0.0.0.0' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_separator_two_env_var(self): os.environ[self.env_var1] = "1value" os.environ[self.env_var2] = "2values" test_data = ''' test1: data0: !TEST ${ENV_TAG1:NoHtnnmEuluGp2boPvGQkGrXqTAtBvIVz9VRmV65}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': '1value/somethingelse/2values', 'data1': '2values' } } self.assertDictEqual( config, expected_config ) def test_parse_config_two_env_var_extra_chars_in_env_var(self): os.environ[self.env_var2] = "1value" os.environ[self.env_var4] = "2values" test_data = ''' test1: data0: !TEST ${ENV*)__TAG101sfdarg:NoHtnnmEuluGp2boPvGQkGrXqTAtBvIVz.,hujn+000-!!#9VRmV65}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2:0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=':') expected_config = { 'test1': { 'data0': '2values/somethingelse/1value', 'data1': '1value' } } self.assertDictEqual( config, expected_config ) def test_parse_config_default_sep_blank_value_error(self): os.environ[self.env_var2] = "1value" test_data = ''' test1: data0: !TEST ${ENV*)__TAG101sfdarg dsrme__dfweggrsg}/somethingelse/${ENV_TAG2} data1: !TEST ${ENV_TAG2 0.0.0.0} ''' config = parse_config(data=test_data, tag='!TEST', default_sep=' ') expected_config = { 'test1': { 'data0': 'dsrme__dfweggrsg/somethingelse/1value', 'data1': '1value' } } self.assertDictEqual( config, expected_config ) def test_default_loader(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !ENV ${ENV_TAG1} data1: !ENV ${ENV_TAG2} ''' config = parse_config(data=test_data, loader=yaml.UnsafeLoader) expected_config = { 'test1': { 'data0': 'it works!', 'data1': 'this works too!' } } self.assertDictEqual( config, expected_config ) def test_default_unsafe_loader_no_env_vars(self): unsafe_load_instance = UnsafeLoadTest() test_data = ''' !!python/object:tests.pyaml_env_tests.test_parse_config.UnsafeLoadTest data0: it works! data1: this works too! ''' config = parse_config(data=test_data, loader=yaml.UnsafeLoader) self.assertIsInstance( config, UnsafeLoadTest ) self.assertEqual(config.data0, unsafe_load_instance.data0) self.assertEqual(config.data1, unsafe_load_instance.data1) def test_default_unsafe_loader_env_vars(self): os.environ[self.env_var1] = 'it works differently!' os.environ[self.env_var2] = 'this works too, new value!' test_data = ''' !!python/object:tests.pyaml_env_tests.test_parse_config.UnsafeLoadTest data0: !ENV ${ENV_TAG1} data1: !ENV ${ENV_TAG2} ''' config = parse_config(data=test_data, loader=yaml.UnsafeLoader) self.assertIsInstance( config, UnsafeLoadTest ) self.assertEqual(config.data0, os.environ[self.env_var1]) self.assertEqual(config.data1, os.environ[self.env_var2]) def test_default_unsafe_loader_env_vars_default_value(self): os.environ[self.env_var1] = 'it works differently!' test_data = ''' !!python/object:tests.pyaml_env_tests.test_parse_config.UnsafeLoadTest data0: !ENV ${ENV_TAG1} data1: !ENV ${ENV_TAG2:default} ''' config = parse_config(data=test_data, loader=yaml.UnsafeLoader) self.assertIsInstance( config, UnsafeLoadTest ) self.assertEqual(config.data0, os.environ[self.env_var1]) self.assertEqual(config.data1, "default") def test_parse_config_different_tag(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST ${ENV_TAG1} data1: !TEST ${ENV_TAG2} ''' expected = { 'test1': { 'data0': os.environ[self.env_var1], 'data1': os.environ[self.env_var2] } } result = parse_config(data=test_data, tag='!TEST') self.assertDictEqual(result, expected) def test_parse_config_different_tag_error(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST2 ${ENV_TAG1} data1: !TEST2 ${ENV_TAG2} ''' with self.assertRaises(ConstructorError) as ce: _ = parse_config(data=test_data, tag='!TEST') self.assertTrue( "could not determine a constructor for the tag '!TEST2'" in str(ce.exception.problem) ) def test_parse_config_only_tag_env_vars_resolved(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST2 test1${ENV_TAG1} data1: ${ENV_TAG2} ''' expected = { 'test1': { 'data0': f'test1{os.environ[self.env_var1]}', 'data1': '${ENV_TAG2}' } } # because None is used as a tag in one of the tests, it messes up expected behavior # remove None from implicit resolvers: if None in yaml.SafeLoader.yaml_implicit_resolvers: del yaml.SafeLoader.yaml_implicit_resolvers[None] # only ENV_TAG1 should be parsed because of !TEST2 tag result = parse_config(data=test_data, tag='!TEST2') self.assertDictEqual(result, expected) def test_parse_config_no_tag_all_resolved(self): os.environ[self.env_var1] = 'it works!' os.environ[self.env_var2] = 'this works too!' test_data = ''' test1: data0: !TEST2 test1${ENV_TAG1} data1: ${ENV_TAG2} ''' expected = { 'test1': { 'data0': f'test1{os.environ[self.env_var1]}', 'data1': 'this works too!' } } # all environment variables will be parsed result = parse_config(data=test_data, tag=None) self.assertDictEqual(result, expected) self.assertDictEqual(result, expected) def test_numeric_values_with_type_defined(self): os.environ[self.env_var1] = "1024" test_data = ''' data0: !TAG ${ENV_TAG1} data1: !TAG tag:yaml.org,2002:float ${ENV_TAG2:27017} data2: !!float 1024 data3: !TAG ${ENV_TAG2:some_value} data4: !TAG tag:yaml.org,2002:bool ${ENV_TAG2:false} ''' config = parse_config(data=test_data, tag='!TAG') print(config) self.assertIsInstance(config['data2'], float) self.assertIsInstance(config['data3'], str) self.assertIsInstance(config['data1'], float) self.assertIsInstance(config['data4'], bool) self.assertIsInstance(config['data0'], str) self.assertEqual(config['data0'], os.environ[self.env_var1]) self.assertEqual(config['data2'], float(os.environ[self.env_var1])) self.assertEqual(config['data1'], 27017.0) self.assertEqual(config['data3'], "some_value") self.assertEqual(config['data4'], False)