pax_global_header00006660000000000000000000000064135144507170014521gustar00rootroot0000000000000052 comment=26a67e9419d96b7f92871e8b93dba00306c5df0b .gitignore000066400000000000000000000006101351445071700130520ustar00rootroot00000000000000# ===[ APP ]=== # # ===[ PYTHON PACKAGE ]=== # /build/ /dist/ /MANIFEST /*.egg/ /*.egg-info/ # ===[ OTHER ]=== # # IDE Projects .idea .nbproject .project *.sublime-project # Temps *~ *.tmp *.bak *.swp *.kate-swp *.DS_Store Thumbs.db # Utils /.tox/ .sass-cache/ .coverage # Generated __pycache__ *.py[cod] *.pot *.mo # Runtime /*.log /*.pid # ===[ EXCLUDES ]=== # !.gitkeep !.htaccess .python-version000066400000000000000000000000451351445071700140710ustar00rootroot00000000000000j2cli 2.7.16 3.4.9 3.5.6 3.6.8 3.7.2 .travis.yml000066400000000000000000000012211351445071700131720ustar00rootroot00000000000000os: linux sudo: false language: python matrix: include: - python: 2.7 env: TOXENV=py27 - python: 3.4 env: TOXENV=py34 - python: 3.5 env: TOXENV=py35 - python: 3.6 env: TOXENV=py36 - python: 3.7-dev env: TOXENV=py37 - python: pypy env: TOXENV=pypy - python: pypy3 env: TOXENV=pypy - {python: 3.6, env: TOXENV=py36-pyyaml5.1} - {python: 3.6, env: TOXENV=py36-pyyaml3.13} - {python: 3.6, env: TOXENV=py36-pyyaml3.12} - {python: 3.6, env: TOXENV=py36-pyyaml3.11} - {python: 3.6, env: TOXENV=py36-pyyaml3.10} install: - pip install tox cache: - pip script: - tox CHANGELOG.md000066400000000000000000000024301351445071700126750ustar00rootroot00000000000000## 0.3.12 (2019-08-18) * Fix: use `env` format from stdin ## 0.3.10 (2019-06-07) * New: `env()` is now available as a function * New: can now customize the `Environment` object * Fixed documentation ## 0.3.9 (2019-06-04) * New: customize.py that lets you customize :) * Fixed a bug with setup.py and yaml ## 0.3.8 (2019-04-29) * Enabled Jinja2 extensions: i18n, do, loopcontrols ## 0.3.7 (2019-04-23) * The new `{{ VAR_NAME |env }}` filter lets you use environment variables in every template. ## 0.3.6 (2019-03-21) * Fixed support for Python 2.6 * Dropped Python 2.6 from unit-tests~~~~ * Fixed a warning issued by PyYAML. See [issue #33](https://github.com/kolypto/j2cli/issues/33) ## 0.3.5 (2019-01-03) * New option: `--undefined` that allows undefined variables * Fix: unicode support in environment variables ## 0.3.4 (2018-12-26) * `-o outfile` option writes to a file ## 0.3.3 * Python 3 support. Supported versions: 2.6, 2.7, 3.6, 3.7 * New CLI option: `--import-env` that imports environment variables into the template * New options: `--filters` and `--tests` that import custom Jinja2 filters from a Python file * Fix: trailing newline is not removed anymore * Fix: env vars with "=" in values are now parsed correctly * Fix: now unicode templates & contexts are fully supported LICENSE000066400000000000000000000024511351445071700120740ustar00rootroot00000000000000Copyright (c) 2014, Mark Vartanyan All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. MANIFEST.in000066400000000000000000000000611351445071700126200ustar00rootroot00000000000000include README.* include LICENSE include tox.ini Makefile000066400000000000000000000011121351445071700125200ustar00rootroot00000000000000all: SHELL := /bin/bash # Package .PHONY: clean clean: @rm -rf build/ dist/ *.egg-info/ README.md README.rst @pip install -e . # have to reinstall because we are using self README.md: $(shell find j2cli/) $(wildcard misc/_doc/**) @python misc/_doc/README.py | python j2cli/__init__.py -f json -o $@ misc/_doc/README.md.j2 .PHONY: build publish-test publish build: README.md @./setup.py build sdist bdist_wheel publish-test: README.md @twine upload --repository pypitest dist/* publish: README.md @twine upload dist/* .PHONY: test test-tox test: @nosetests test-tox: @tox README.md000066400000000000000000000226011351445071700123450ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) [![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) j2cli - Jinja2 Command-Line Tool ================================ `j2cli` is a command-line tool for templating in shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. Features: * Jinja2 templating * INI, YAML, JSON data sources supported * Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) ## Installation ``` pip install j2cli ``` To enable the YAML support with [pyyaml](http://pyyaml.org/): ``` pip install j2cli[yaml] ``` ## Tutorial Suppose, you want to have an nginx configuration file template, `nginx.j2`: ```jinja2 server { listen 80; server_name {{ nginx.hostname }}; root {{ nginx.webroot }}; index index.htm; } ``` And you have a JSON file with the data, `nginx.json`: ```json { "nginx":{ "hostname": "localhost", "webroot": "/var/www/project" } } ``` This is how you render it into a working configuration file: ```bash $ j2 -f json nginx.j2 nginx.json > nginx.conf ``` The output is saved to `nginx.conf`: ``` server { listen 80; server_name localhost; root /var/www/project; index index.htm; } ``` Alternatively, you can use the `-o nginx.conf` option. ## Tutorial with environment variables Suppose, you have a very simple template, `person.xml`: ```jinja2 {{ name }}{{ age }} ``` What is the easiest way to use j2 here? Use environment variables in your bash script: ```bash $ export name=Andrew $ export age=31 $ j2 /tmp/person.xml Andrew31 ``` ## Using environment variables Even when you use yaml or json as the data source, you can always access environment variables using the `env()` function: ```jinja2 Username: {{ login }} Password: {{ env("APP_PASSWORD") }} ``` ## Usage Compile a template using INI-file data source: $ j2 config.j2 data.ini Compile using JSON data source: $ j2 config.j2 data.json Compile using YAML data source (requires PyYAML): $ j2 config.j2 data.yaml Compile using JSON data on stdin: $ curl http://example.com/service.json | j2 --format=json config.j2 Compile using environment variables (hello Docker!): $ j2 config.j2 Or even read environment variables from a file: $ j2 --format=env config.j2 data.env Or pipe it: (note that you'll have to use the "-" in this particular case): $ j2 --format=env config.j2 - < data.env # Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render * `data`: (optional) path to the data used for rendering. The default is `-`: use stdin. Specify it explicitly when using env! Options: * `--format, -f`: format for the data file. The default is `?`: guess from file extension. * `--import-env VAR, -e EVAR`: import all environment variables into the template as `VAR`. To import environment variables into the global scope, give it an empty string: `--import-env=`. (This will overwrite any existing variables!) * `-o outfile`: Write rendered template to a file * `--undefined`: Allow undefined variables to be used in templates (no error will be raised) * `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. Will load all top-level functions and register them as filters. This option can be used multiple times to import several files. * `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. * `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. See [Customization](#customization) for more info. There is some special behavior with environment variables: * When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables * When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` ## Formats ### env Data input from environment variables. Render directly from the current environment variable values: $ j2 config.j2 Or alternatively, read the values from a dotenv file: ``` NGINX_HOSTNAME=localhost NGINX_WEBROOT=/var/www/project NGINX_LOGS=/var/log/nginx/ ``` And render with: $ j2 config.j2 data.env $ env | j2 --format=env config.j2 If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: $ j2 config.j2 - < data.env ### ini INI data input format. data.ini: ``` [nginx] hostname=localhost webroot=/var/www/project logs=/var/log/nginx/ ``` Usage: $ j2 config.j2 data.ini $ cat data.ini | j2 --format=ini config.j2 ### json JSON data input format data.json: ``` { "nginx":{ "hostname": "localhost", "webroot": "/var/www/project", "logs": "/var/log/nginx/" } } ``` Usage: $ j2 config.j2 data.json $ cat data.json | j2 --format=ini config.j2 ### yaml YAML data input format. data.yaml: ``` nginx: hostname: localhost webroot: /var/www/project logs: /var/log/nginx ``` Usage: $ j2 config.j2 data.yml $ cat data.yml | j2 --format=yaml config.j2 Extras ====== ## Filters ### `docker_link(value, format='{addr}:{port}')` Given a Docker Link environment variable value, format it into something else. This first parses a Docker Link value like this: DB_PORT=tcp://172.17.0.5:5432 Into a dict: ```python { 'proto': 'tcp', 'addr': '172.17.0.5', 'port': '5432' } ``` And then uses `format` to format it, where the default format is '{addr}:{port}'. More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) ### `env(varname, default=None)` Use an environment variable's value inside your template. This filter is available even when your data source is something other that the environment. Example: ```jinja2 User: {{ user_login }} Pass: {{ "USER_PASSWORD"|env }} ``` You can provide the default value: ```jinja2 Pass: {{ "USER_PASSWORD"|env("-none-") }} ``` For your convenience, it's also available as a function: ```jinja2 User: {{ user_login }} Pass: {{ env("USER_PASSWORD") }} ``` Notice that there must be quotes around the environment variable name Customization ============= j2cli now allows you to customize the way the application is initialized: * Pass additional keywords to Jinja2 environment * Modify the context before it's used for rendering * Register custom filters and tests This is done through *hooks* that you implement in a customization file in Python language. Just plain functions at the module level. The following hooks are available: * `j2_environment_params() -> dict`: returns a `dict` of additional parameters for [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). * `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. * `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be used for template rendering. You can do all sorts of pre-processing here. * `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 * `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 All of them are optional. The example customization.py file for your reference: ```python # # Example customize.py file for j2cli # Contains potional hooks that modify the way j2cli is initialized def j2_environment_params(): """ Extra parameters for the Jinja2 Environment """ # Jinja2 Environment configuration # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment return dict( # Just some examples # Change block start/end strings block_start_string='<%', block_end_string='%>', # Change variable strings variable_start_string='<<', variable_end_string='>>', # Remove whitespace around blocks trim_blocks=True, lstrip_blocks=True, # Enable line statements: # http://jinja.pocoo.org/docs/2.10/templates/#line-statements line_statement_prefix='#', # Keep \n at the end of a file keep_trailing_newline=True, # Enable custom extensions # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions extensions=('jinja2.ext.i18n',), ) def j2_environment(env): """ Modify Jinja2 environment :param env: jinja2.environment.Environment :rtype: jinja2.environment.Environment """ env.globals.update( my_function=lambda v: 'my function says "{}"'.format(v) ) return env def alter_context(context): """ Modify the context and return it """ # An extra variable context['ADD'] = '127' return context def extra_filters(): """ Declare some custom filters. Returns: dict(name = function) """ return dict( # Example: {{ var | parentheses }} parentheses=lambda t: '(' + t + ')', ) def extra_tests(): """ Declare some custom tests Returns: dict(name = function) """ return dict( # Example: {% if a|int is custom_odd %}odd{% endif %} custom_odd=lambda n: True if (n % 2) else False ) # ``` j2cli/000077500000000000000000000000001351445071700120705ustar00rootroot00000000000000j2cli/__init__.py000066400000000000000000000004051351445071700142000ustar00rootroot00000000000000#! /usr/bin/env python """ j2cli main file """ import pkg_resources __author__ = "Mark Vartanyan" __email__ = "kolypto@gmail.com" __version__ = pkg_resources.get_distribution('j2cli').version from j2cli.cli import main if __name__ == '__main__': main() j2cli/cli.py000066400000000000000000000214011351445071700132070ustar00rootroot00000000000000import io, os, sys import argparse import jinja2 import jinja2.loaders from . import __version__ import imp, inspect from .context import read_context_data, FORMATS from .extras import filters from .extras.customize import CustomizationModule class FilePathLoader(jinja2.BaseLoader): """ Custom Jinja2 template loader which just loads a single template file """ def __init__(self, cwd, encoding='utf-8'): self.cwd = cwd self.encoding = encoding def get_source(self, environment, template): # Path filename = os.path.join(self.cwd, template) # Read try: with io.open(template, 'rt', encoding=self.encoding) as f: contents = f.read() except IOError: raise jinja2.TemplateNotFound(template) # Finish uptodate = lambda: False return contents, filename, uptodate class Jinja2TemplateRenderer(object): """ Template renderer """ ENABLED_EXTENSIONS=( 'jinja2.ext.i18n', 'jinja2.ext.do', 'jinja2.ext.loopcontrols', ) def __init__(self, cwd, allow_undefined, j2_env_params): # Custom env params j2_env_params.setdefault('keep_trailing_newline', True) j2_env_params.setdefault('undefined', jinja2.Undefined if allow_undefined else jinja2.StrictUndefined) j2_env_params.setdefault('extensions', self.ENABLED_EXTENSIONS) j2_env_params.setdefault('loader', FilePathLoader(cwd)) # Environment self._env = jinja2.Environment(**j2_env_params) self._env.globals.update(dict( env=filters.env )) def register_filters(self, filters): self._env.filters.update(filters) def register_tests(self, tests): self._env.tests.update(tests) def import_filters(self, filename): self.register_filters(self._import_functions(filename)) def import_tests(self, filename): self.register_tests(self._import_functions(filename)) def _import_functions(self, filename): m = imp.load_source('imported-funcs', filename) return dict((name, func) for name, func in inspect.getmembers(m) if inspect.isfunction(func)) def render(self, template_path, context): """ Render a template :param template_path: Path to the template file :type template_path: basestring :param context: Template data :type context: dict :return: Rendered template :rtype: basestring """ return self._env \ .get_template(template_path) \ .render(context) \ .encode('utf-8') def render_command(cwd, environ, stdin, argv): """ Pure render command :param cwd: Current working directory (to search for the files) :type cwd: basestring :param environ: Environment variables :type environ: dict :param stdin: Stdin stream :type stdin: file :param argv: Command-line arguments :type argv: list :return: Rendered template :rtype: basestring """ parser = argparse.ArgumentParser( prog='j2', description='Command-line interface to Jinja2 for templating in shell scripts.', epilog='' ) parser.add_argument('-v', '--version', action='version', version='j2cli {0}, Jinja2 {1}'.format(__version__, jinja2.__version__)) parser.add_argument('-f', '--format', default='?', help='Input data format', choices=['?'] + list(FORMATS.keys())) parser.add_argument('-e', '--import-env', default=None, metavar='VAR', dest='import_env', help='Import environment variables as `var` variable. Use empty string to import into the top level') parser.add_argument('--filters', nargs='+', default=[], metavar='python-file', dest='filters', help='Load custom Jinja2 filters from a Python file: all top-level functions are imported.') parser.add_argument('--tests', nargs='+', default=[], metavar='python-file', dest='tests', help='Load custom Jinja2 tests from a Python file.') parser.add_argument('--customize', default=None, metavar='python-file.py', dest='customize', help='A Python file that implements hooks to fine-tune the j2cli behavior') parser.add_argument('--undefined', action='store_true', dest='undefined', help='Allow undefined variables to be used in templates (no error will be raised)') parser.add_argument('-o', metavar='outfile', dest='output_file', help="Output to a file instead of stdout") parser.add_argument('template', help='Template file to process') parser.add_argument('data', nargs='?', default=None, help='Input data file path; "-" to use stdin') args = parser.parse_args(argv) # Input: guess format if args.format == '?': if args.data is None or args.data == '-': args.format = 'env' else: args.format = { '.ini': 'ini', '.json': 'json', '.yml': 'yaml', '.yaml': 'yaml', '.env': 'env' }[os.path.splitext(args.data)[1]] # Input: data # We always expect a file; # unless the user wants 'env', and there's no input file provided. if args.format == 'env': # With the "env" format, if no dotenv filename is provided, we have two options: # either the user wants to use the current environment, or he's feeding a dotenv file at stdin. # Depending on whether we have data at stdin, we'll need to choose between the two. # # The problem is that in Linux, you can't reliably determine whether there is any data at stdin: # some environments would open the descriptor even though they're not going to feed any data in. # That's why many applications would ask you to explicitly specify a '-' when stdin should be used. # # And this is what we're going to do here as well. # The script, however, would give the user a hint that they should use '-' if args.data == '-': input_data_f = stdin elif args.data == None: input_data_f = None else: input_data_f = open(args.data) else: input_data_f = stdin if args.data is None or args.data == '-' else open(args.data) # Python 2: Encode environment variables as unicode if sys.version_info[0] == 2 and args.format == 'env': environ = dict((k.decode('utf-8'), v.decode('utf-8')) for k, v in environ.items()) # Customization if args.customize is not None: customize = CustomizationModule( imp.load_source('customize-module', args.customize) ) else: customize = CustomizationModule(None) # Read data context = read_context_data( args.format, input_data_f, environ, args.import_env ) context = customize.alter_context(context) # Renderer renderer = Jinja2TemplateRenderer(cwd, args.undefined, j2_env_params=customize.j2_environment_params()) customize.j2_environment(renderer._env) # Filters, Tests renderer.register_filters({ 'docker_link': filters.docker_link, 'env': filters.env, }) for fname in args.filters: renderer.import_filters(fname) for fname in args.tests: renderer.import_tests(fname) renderer.register_filters(customize.extra_filters()) renderer.register_tests(customize.extra_tests()) # Render try: result = renderer.render(args.template, context) except jinja2.exceptions.UndefinedError as e: # When there's data at stdin, tell the user they should use '-' try: stdin_has_data = stdin is not None and not stdin.isatty() if args.format == 'env' and args.data == None and stdin_has_data: extra_info = ( "\n\n" "If you're trying to pipe a .env file, please run me with a '-' as the data file name:\n" "$ {cmd} {argv} -".format(cmd=os.path.basename(sys.argv[0]), argv=' '.join(sys.argv[1:])) ) e.args = (e.args[0] + extra_info,) + e.args[1:] except: # The above code is so optional that any, ANY, error, is ignored pass # Proceed raise # -o if args.output_file: with io.open(args.output_file, 'wt', encoding='utf-8') as f: f.write(result.decode('utf-8')) f.close() return b'' # Finish return result def main(): """ CLI Entry point """ try: output = render_command( os.getcwd(), os.environ, sys.stdin, sys.argv[1:] ) except SystemExit: return 1 outstream = getattr(sys.stdout, 'buffer', sys.stdout) outstream.write(output) j2cli/context.py000066400000000000000000000104711351445071700141310ustar00rootroot00000000000000import sys # Patch basestring in for python 3 compat try: basestring except NameError: basestring = str #region Parsers def _parse_ini(data_string): """ INI data input format. data.ini: ``` [nginx] hostname=localhost webroot=/var/www/project logs=/var/log/nginx/ ``` Usage: $ j2 config.j2 data.ini $ cat data.ini | j2 --format=ini config.j2 """ from io import StringIO # Override class MyConfigParser(ConfigParser.ConfigParser): def as_dict(self): """ Export as dict :rtype: dict """ d = dict(self._sections) for k in d: d[k] = dict(self._defaults, **d[k]) d[k].pop('__name__', None) return d # Parse ini = MyConfigParser() ini.readfp(ini_file_io(data_string)) # Export return ini.as_dict() def _parse_json(data_string): """ JSON data input format data.json: ``` { "nginx":{ "hostname": "localhost", "webroot": "/var/www/project", "logs": "/var/log/nginx/" } } ``` Usage: $ j2 config.j2 data.json $ cat data.json | j2 --format=ini config.j2 """ return json.loads(data_string) def _parse_yaml(data_string): """ YAML data input format. data.yaml: ``` nginx: hostname: localhost webroot: /var/www/project logs: /var/log/nginx ``` Usage: $ j2 config.j2 data.yml $ cat data.yml | j2 --format=yaml config.j2 """ # Loader try: # PyYAML 5.1 supports FullLoader Loader = yaml.FullLoader except AttributeError: # Have to use SafeLoader for older versions Loader = yaml.SafeLoader # Done return yaml.load(data_string, Loader=Loader) def _parse_env(data_string): """ Data input from environment variables. Render directly from the current environment variable values: $ j2 config.j2 Or alternatively, read the values from a dotenv file: ``` NGINX_HOSTNAME=localhost NGINX_WEBROOT=/var/www/project NGINX_LOGS=/var/log/nginx/ ``` And render with: $ j2 config.j2 data.env $ env | j2 --format=env config.j2 If you're going to pipe a dotenv file into `j2`, you'll need to use "-" as the second argument to explicitly: $ j2 config.j2 - < data.env """ # Parse if isinstance(data_string, basestring): data = filter( lambda l: len(l) == 2 , ( list(map( str.strip, line.split('=', 1) )) for line in data_string.split("\n")) ) else: data = data_string # Finish return data FORMATS = { 'ini': _parse_ini, 'json': _parse_json, 'yaml': _parse_yaml, 'env': _parse_env } #endregion #region Imports # JSON: simplejson | json try: import simplejson as json except ImportError: try: import json except ImportError: del FORMATS['json'] # INI: Python 2 | Python 3 try: import ConfigParser from io import BytesIO as ini_file_io except ImportError: import configparser as ConfigParser from io import StringIO as ini_file_io # YAML try: import yaml except ImportError: del FORMATS['yaml'] #endregion def read_context_data(format, f, environ, import_env=None): """ Read context data into a dictionary :param format: Data format :type format: str :param f: Data file stream, or None (for env) :type f: file|None :param import_env: Variable name, if any, that will contain environment variables of the template. :type import_env: bool|None :return: Dictionary with the context data :rtype: dict """ # Special case: environment variables if format == 'env' and f is None: return _parse_env(environ) # Read data string stream data_string = f.read() # Parse it if format not in FORMATS: raise ValueError('{0} format unavailable'.format(format)) context = FORMATS[format](data_string) # Import environment if import_env is not None: if import_env == '': context.update(environ) else: context[import_env] = environ # Done return context j2cli/extras/000077500000000000000000000000001351445071700133765ustar00rootroot00000000000000j2cli/extras/__init__.py000066400000000000000000000000261351445071700155050ustar00rootroot00000000000000from . import filters j2cli/extras/customize.py000066400000000000000000000017241351445071700157760ustar00rootroot00000000000000 class CustomizationModule(object): """ The interface for customization functions, defined as module-level functions """ def __init__(self, module=None): if module is not None: def howcall(*args): print(args) exit(1) # Import every module function as a method on ourselves for name in self._IMPORTED_METHOD_NAMES: try: setattr(self, name, getattr(module, name)) except AttributeError: pass # stubs def j2_environment_params(self): return {} def j2_environment(self, env): return env def alter_context(self, context): return context def extra_filters(self): return {} def extra_tests(self): return {} _IMPORTED_METHOD_NAMES = [ f.__name__ for f in (j2_environment_params, j2_environment, alter_context, extra_filters, extra_tests)] j2cli/extras/filters.py000066400000000000000000000040601351445071700154200ustar00rootroot00000000000000""" Custom Jinja2 filters """ import os from jinja2 import is_undefined import re def docker_link(value, format='{addr}:{port}'): """ Given a Docker Link environment variable value, format it into something else. This first parses a Docker Link value like this: DB_PORT=tcp://172.17.0.5:5432 Into a dict: ```python { 'proto': 'tcp', 'addr': '172.17.0.5', 'port': '5432' } ``` And then uses `format` to format it, where the default format is '{addr}:{port}'. More info here: [Docker Links](https://docs.docker.com/userguide/dockerlinks/) :param value: Docker link (from an environment variable) :param format: The format to apply. Supported placeholders: `{proto}`, `{addr}`, `{port}` :return: Formatted string """ # pass undefined values on down the pipeline if is_undefined(value): return value # Parse the value m = re.match(r'(?P.+)://' r'(?P.+):' r'(?P.+)$', value) if not m: raise ValueError('The provided value does not seems to be a Docker link: {0}'.format(value)) d = m.groupdict() # Format return format.format(**d) def env(varname, default=None): """ Use an environment variable's value inside your template. This filter is available even when your data source is something other that the environment. Example: ```jinja2 User: {{ user_login }} Pass: {{ "USER_PASSWORD"|env }} ``` You can provide the default value: ```jinja2 Pass: {{ "USER_PASSWORD"|env("-none-") }} ``` For your convenience, it's also available as a function: ```jinja2 User: {{ user_login }} Pass: {{ env("USER_PASSWORD") }} ``` Notice that there must be quotes around the environment variable name """ if default is not None: # With the default, there's never an error return os.getenv(varname, default) else: # Raise KeyError when not provided return os.environ[varname] misc/000077500000000000000000000000001351445071700120205ustar00rootroot00000000000000misc/_doc/000077500000000000000000000000001351445071700127245ustar00rootroot00000000000000misc/_doc/README.md.j2000066400000000000000000000132201351445071700145130ustar00rootroot00000000000000[![Build Status](https://travis-ci.org/kolypto/j2cli.svg)](https://travis-ci.org/kolypto/j2cli) [![Pythons](https://img.shields.io/badge/python-2.6%20%7C%202.7%20%7C%203.4%E2%80%933.7%20%7C%20pypy-blue.svg)](.travis.yml) j2cli - Jinja2 Command-Line Tool ================================ `j2cli` is a command-line tool for templating in shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. Features: * Jinja2 templating * INI, YAML, JSON data sources supported * Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) ## Installation ``` pip install j2cli ``` To enable the YAML support with [pyyaml](http://pyyaml.org/): ``` pip install j2cli[yaml] ``` ## Tutorial Suppose, you want to have an nginx configuration file template, `nginx.j2`: {% raw %}```jinja2 server { listen 80; server_name {{ nginx.hostname }}; root {{ nginx.webroot }}; index index.htm; } ```{% endraw %} And you have a JSON file with the data, `nginx.json`: ```json { "nginx":{ "hostname": "localhost", "webroot": "/var/www/project" } } ``` This is how you render it into a working configuration file: ```bash $ j2 -f json nginx.j2 nginx.json > nginx.conf ``` The output is saved to `nginx.conf`: ``` server { listen 80; server_name localhost; root /var/www/project; index index.htm; } ``` Alternatively, you can use the `-o nginx.conf` option. ## Tutorial with environment variables Suppose, you have a very simple template, `person.xml`: {% raw %}```jinja2 {{ name }}{{ age }} ```{% endraw %} What is the easiest way to use j2 here? Use environment variables in your bash script: ```bash $ export name=Andrew $ export age=31 $ j2 /tmp/person.xml Andrew31 ``` ## Using environment variables Even when you use yaml or json as the data source, you can always access environment variables using the `env()` function: {% raw %}```jinja2 Username: {{ login }} Password: {{ env("APP_PASSWORD") }} ```{% endraw %} ## Usage Compile a template using INI-file data source: $ j2 config.j2 data.ini Compile using JSON data source: $ j2 config.j2 data.json Compile using YAML data source (requires PyYAML): $ j2 config.j2 data.yaml Compile using JSON data on stdin: $ curl http://example.com/service.json | j2 --format=json config.j2 Compile using environment variables (hello Docker!): $ j2 config.j2 Or even read environment variables from a file: $ j2 --format=env config.j2 data.env Or pipe it: (note that you'll have to use the "-" in this particular case): $ j2 --format=env config.j2 - < data.env # Reference `j2` accepts the following arguments: * `template`: Jinja2 template file to render * `data`: (optional) path to the data used for rendering. The default is `-`: use stdin. Specify it explicitly when using env! Options: * `--format, -f`: format for the data file. The default is `?`: guess from file extension. * `--import-env VAR, -e EVAR`: import all environment variables into the template as `VAR`. To import environment variables into the global scope, give it an empty string: `--import-env=`. (This will overwrite any existing variables!) * `-o outfile`: Write rendered template to a file * `--undefined`: Allow undefined variables to be used in templates (no error will be raised) * `--filters filters.py`: Load custom Jinja2 filters and tests from a Python file. Will load all top-level functions and register them as filters. This option can be used multiple times to import several files. * `--tests tests.py`: Load custom Jinja2 filters and tests from a Python file. * `--customize custom.py`: A Python file that implements hooks to fine-tune the j2cli behavior. This is fairly advanced stuff, use it only if you really need to customize the way Jinja2 is initialized. See [Customization](#customization) for more info. There is some special behavior with environment variables: * When `data` is not provided (data is `-`), `--format` defaults to `env` and thus reads environment variables * When `--format=env`, it can read a special "environment variables" file made like this: `env > /tmp/file.env` ## Formats {% for name, format in formats|dictsort() %} ### {{ name }} {{ format.doc }} {% endfor %} Extras ====== ## Filters {% for name, filter in extras.filters|dictsort() %} ### `{{ filter.qsignature }}` {{ filter.doc }} {% endfor %} Customization ============= j2cli now allows you to customize the way the application is initialized: * Pass additional keywords to Jinja2 environment * Modify the context before it's used for rendering * Register custom filters and tests This is done through *hooks* that you implement in a customization file in Python language. Just plain functions at the module level. The following hooks are available: * `j2_environment_params() -> dict`: returns a `dict` of additional parameters for [Jinja2 Environment](http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment). * `j2_environment(env: Environment) -> Environment`: lets you customize the `Environment` object. * `alter_context(context: dict) -> dict`: lets you modify the context variables that are going to be used for template rendering. You can do all sorts of pre-processing here. * `extra_filters() -> dict`: returns a `dict` with extra filters for Jinja2 * `extra_tests() -> dict`: returns a `dict` with extra tests for Jinja2 All of them are optional. The example customization.py file for your reference: ```python {% include "tests/resources/customize.py" %} ``` misc/_doc/README.py000077500000000000000000000011111351445071700142300ustar00rootroot00000000000000#! /usr/bin/env python import json import inspect from exdoc import doc, getmembers import j2cli import j2cli.context import j2cli.extras.filters README = { 'formats': { name: doc(f) for name, f in j2cli.context.FORMATS.items() }, 'extras': { 'filters': {k: doc(v) for k, v in getmembers(j2cli.extras.filters) if inspect.isfunction(v) and inspect.getmodule(v) is j2cli.extras.filters} } } assert 'yaml' in README['formats'], 'Looks like the YAML library is not installed!' print(json.dumps(README)) requirements-dev.txt000066400000000000000000000000211351445071700151160ustar00rootroot00000000000000wheel nose exdoc setup.cfg000066400000000000000000000000771351445071700127120ustar00rootroot00000000000000[bdist_wheel] universal = 1 [metadata] license_file = LICENSE setup.py000077500000000000000000000037121351445071700126050ustar00rootroot00000000000000#!/usr/bin/env python """ j2cli - Jinja2 Command-Line Tool ================================ `j2cli` is a command-line tool for templating in shell-scripts, leveraging the [Jinja2](http://jinja.pocoo.org/docs/) library. Features: * Jinja2 templating * INI, YAML, JSON data sources supported * Allows the use of environment variables in templates! Hello [Docker](http://www.docker.com/) :) Inspired by [mattrobenolt/jinja2-cli](https://github.com/mattrobenolt/jinja2-cli) """ from setuptools import setup, find_packages import sys # PyYAML 3.11 was the last to support Python 2.6 # This code limits pyyaml version for older pythons pyyaml_version = 'pyyaml >= 3.10' # fresh if sys.version_info[:2] == (2, 6): pyyaml_version = 'pyyaml<=3.11' setup( name='j2cli', version='0.3.12b', author='Mark Vartanyan', author_email='kolypto@gmail.com', url='https://github.com/kolypto/j2cli', license='BSD', description='Command-line interface to Jinja2 for templating in shell scripts.', long_description=__doc__, # can't do open('README.md').read() because we're describing self long_description_content_type='text/markdown', keywords=['Jinja2', 'templating', 'command-line', 'CLI'], packages=find_packages(), scripts=[], entry_points={ 'console_scripts': [ 'j2 = j2cli:main', ] }, install_requires=[ 'jinja2 >= 2.7.2', ], extras_require={ 'yaml': [pyyaml_version,] }, include_package_data=True, zip_safe=False, test_suite='nose.collector', platforms='any', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'Intended Audience :: System Administrators', 'Operating System :: OS Independent', 'Topic :: Software Development', 'Natural Language :: English', 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 3', ], ) tests/000077500000000000000000000000001351445071700122275ustar00rootroot00000000000000tests/render-test.py000066400000000000000000000241011351445071700150330ustar00rootroot00000000000000# -*- coding: utf-8 -*- from __future__ import unicode_literals import unittest import os, sys, io, os.path, tempfile from copy import copy from contextlib import contextmanager from jinja2.exceptions import UndefinedError from j2cli.cli import render_command @contextmanager def mktemp(contents): """ Create a temporary file with the given contents, and yield its path """ _, path = tempfile.mkstemp() fp = io.open(path, 'wt+', encoding='utf-8') fp.write(contents) fp.flush() try: yield path finally: fp.close() os.unlink(path) @contextmanager def mock_environ(new_env): old_env = copy(os.environ) os.environ.update(new_env) yield os.environ.clear() os.environ.update(old_env) class RenderTest(unittest.TestCase): def setUp(self): os.chdir( os.path.dirname(__file__) ) def _testme(self, argv, expected_output, stdin=None, env=None): """ Helper test shortcut """ with mock_environ(env or {}): result = render_command(os.getcwd(), env or {}, stdin, argv) if isinstance(result, bytes): result = result.decode('utf-8') self.assertEqual(result, expected_output) #: The expected output expected_output = """server { listen 80; server_name localhost; root /var/www/project; index index.htm; access_log /var/log/nginx//http.access.log combined; error_log /var/log/nginx//http.error.log; } """ def _testme_std(self, argv, stdin=None, env=None): self._testme(argv, self.expected_output, stdin, env) def test_ini(self): # Filename self._testme_std(['resources/nginx.j2', 'resources/data.ini']) # Format self._testme_std(['--format=ini', 'resources/nginx.j2', 'resources/data.ini']) # Stdin self._testme_std(['--format=ini', 'resources/nginx.j2'], stdin=open('resources/data.ini')) self._testme_std(['--format=ini', 'resources/nginx.j2', '-'], stdin=open('resources/data.ini')) def test_json(self): # Filename self._testme_std(['resources/nginx.j2', 'resources/data.json']) # Format self._testme_std(['--format=json', 'resources/nginx.j2', 'resources/data.json']) # Stdin self._testme_std(['--format=json', 'resources/nginx.j2'], stdin=open('resources/data.json')) self._testme_std(['--format=json', 'resources/nginx.j2', '-'], stdin=open('resources/data.json')) def test_yaml(self): try: import yaml except ImportError: raise unittest.SkipTest('Yaml lib not installed') # Filename self._testme_std(['resources/nginx.j2', 'resources/data.yml']) self._testme_std(['resources/nginx.j2', 'resources/data.yaml']) # Format self._testme_std(['--format=yaml', 'resources/nginx.j2', 'resources/data.yml']) # Stdin self._testme_std(['--format=yaml', 'resources/nginx.j2'], stdin=open('resources/data.yml')) self._testme_std(['--format=yaml', 'resources/nginx.j2', '-'], stdin=open('resources/data.yml')) def test_env(self): # Filename self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) # Format self._testme_std(['--format=env', 'resources/nginx-env.j2', 'resources/data.env']) self._testme_std([ 'resources/nginx-env.j2', 'resources/data.env']) # Stdin self._testme_std(['--format=env', 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) self._testme_std([ 'resources/nginx-env.j2', '-'], stdin=open('resources/data.env')) # Environment! # In this case, it's not explicitly provided, but implicitly gotten from the environment env = dict(NGINX_HOSTNAME='localhost', NGINX_WEBROOT='/var/www/project', NGINX_LOGS='/var/log/nginx/') self._testme_std(['--format=env', 'resources/nginx-env.j2'], env=env) self._testme_std([ 'resources/nginx-env.j2'], env=env) def test_import_env(self): # Import environment into a variable with mktemp('{{ a }}/{{ env.B }}') as template: with mktemp('{"a":1}') as context: self._testme(['--format=json', '--import-env=env', template, context], '1/2', env=dict(B='2')) # Import environment into global scope with mktemp('{{ a }}/{{ B }}') as template: with mktemp('{"a":1,"B":1}') as context: self._testme(['--format=json', '--import-env=', template, context], '1/2', env=dict(B='2')) def test_env_file__equals_sign_in_value(self): # Test whether environment variables with "=" in the value are parsed correctly with mktemp('{{ A|default('') }}/{{ B }}/{{ C }}') as template: with mktemp('A\nB=1\nC=val=1\n') as context: self._testme(['--format=env', template, context], '/1/val=1') def test_unicode(self): # Test how unicode is handled # I'm using Russian language for unicode :) with mktemp('Проверка {{ a }} связи!') as template: with mktemp('{"a": "широкополосной"}') as context: self._testme(['--format=json', template, context], 'Проверка широкополосной связи!') # Test case from issue #17: unicode environment variables if sys.version_info[0] == 2: # Python 2: environment variables are bytes self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=b'J\xc3\xbcrgen')) else: # Python 3: environment variables are unicode strings self._testme(['resources/name.j2'], u'Hello Jürgen!\n', env=dict(name=u'Jürgen')) def test_filters__env(self): with mktemp('user_login: kolypto') as yml_file: with mktemp('{{ user_login }}:{{ "USER_PASS"|env }}') as template: # Test: template with an env variable self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) # environment cleaned up assert 'USER_PASS' not in os.environ # Test: KeyError with self.assertRaises(KeyError): self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict()) # Test: default with mktemp('{{ user_login }}:{{ "USER_PASS"|env("-none-") }}') as template: self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) # Test: using as a function with mktemp('{{ user_login }}:{{ env("USER_PASS") }}') as template: self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) with self.assertRaises(KeyError): # Variable not set self._testme(['--format=yaml', template, yml_file], '', env=dict()) # Test: using as a function, with a default with mktemp('{{ user_login }}:{{ env("USER_PASS", "-none-") }}') as template: self._testme(['--format=yaml', template, yml_file], 'kolypto:qwerty123', env=dict(USER_PASS='qwerty123')) self._testme(['--format=yaml', template, yml_file], 'kolypto:-none-', env=dict()) def test_custom_filters(self): with mktemp('{{ a|parentheses }}') as template: self._testme(['--format=env', '--filters=resources/custom_filters.py', template], '(1)', env=dict(a='1')) def test_custom_tests(self): with mktemp('{% if a|int is custom_odd %}odd{% endif %}') as template: self._testme(['--format=env', '--tests=resources/custom_tests.py', template], 'odd', env=dict(a='1')) def test_output_file(self): with mktemp('{{ a }}') as template: try: self._testme(['-o', '/tmp/j2-out', template], '', env=dict(a='123')) self.assertEqual('123', io.open('/tmp/j2-out', 'r').read()) finally: os.unlink('/tmp/j2-out') def test_undefined(self): """ Test --undefined """ # `name` undefined: error self.assertRaises(UndefinedError, self._testme, ['resources/name.j2'], u'Hello !\n', env=dict()) # `name` undefined: no error self._testme(['--undefined', 'resources/name.j2'], u'Hello !\n', env=dict()) def test_jinja2_extensions(self): """ Test that an extension is enabled """ with mktemp('{% do [] %}') as template: # `do` tag is an extension self._testme([template], '') def test_customize(self): """ Test --customize """ # Test: j2_environment_params() # Custom tag start/end with mktemp('<% if 1 %>1<% else %>2<% endif %>') as template: self._testme(['--customize=resources/customize.py', template], '1') # Test: j2_environment() # custom function: my_function with mktemp('<< my_function("hey") >>') as template: self._testme(['--customize=resources/customize.py', template], 'my function says "hey"') # Test: alter_context() # Extra variable: ADD=127 with mktemp('<< ADD >>') as template: self._testme(['--customize=resources/customize.py', template], '127') # Test: extra_filters() with mktemp('<< ADD|parentheses >>') as template: self._testme(['--customize=resources/customize.py', template], '(127)') # Test: extra_tests() with mktemp('<% if ADD|int is custom_odd %>odd<% endif %>') as template: self._testme(['--customize=resources/customize.py', template], 'odd') # reset # otherwise it will load the same module even though its name has changed del sys.modules['customize-module'] # Test: no hooks in a file # Got to restore to the original configuration and use {% %} again with mktemp('{% if 1 %}1{% endif %}') as template: self._testme(['--customize=render-test.py', template], '1') tests/resources/000077500000000000000000000000001351445071700142415ustar00rootroot00000000000000tests/resources/custom_filters.py000066400000000000000000000000561351445071700176560ustar00rootroot00000000000000 def parentheses(t): return '(' + t + ')' tests/resources/custom_tests.py000066400000000000000000000000721351445071700173460ustar00rootroot00000000000000 def custom_odd(n): return True if (n % 2) else False tests/resources/customize.py000066400000000000000000000035771351445071700166510ustar00rootroot00000000000000# {% raw %} # Example customize.py file for j2cli # Contains potional hooks that modify the way j2cli is initialized def j2_environment_params(): """ Extra parameters for the Jinja2 Environment """ # Jinja2 Environment configuration # http://jinja.pocoo.org/docs/2.10/api/#jinja2.Environment return dict( # Just some examples # Change block start/end strings block_start_string='<%', block_end_string='%>', # Change variable strings variable_start_string='<<', variable_end_string='>>', # Remove whitespace around blocks trim_blocks=True, lstrip_blocks=True, # Enable line statements: # http://jinja.pocoo.org/docs/2.10/templates/#line-statements line_statement_prefix='#', # Keep \n at the end of a file keep_trailing_newline=True, # Enable custom extensions # http://jinja.pocoo.org/docs/2.10/extensions/#jinja-extensions extensions=('jinja2.ext.i18n',), ) def j2_environment(env): """ Modify Jinja2 environment :param env: jinja2.environment.Environment :rtype: jinja2.environment.Environment """ env.globals.update( my_function=lambda v: 'my function says "{}"'.format(v) ) return env def alter_context(context): """ Modify the context and return it """ # An extra variable context['ADD'] = '127' return context def extra_filters(): """ Declare some custom filters. Returns: dict(name = function) """ return dict( # Example: {{ var | parentheses }} parentheses=lambda t: '(' + t + ')', ) def extra_tests(): """ Declare some custom tests Returns: dict(name = function) """ return dict( # Example: {% if a|int is custom_odd %}odd{% endif %} custom_odd=lambda n: True if (n % 2) else False ) # {% endraw %} tests/resources/data-empty.env000066400000000000000000000000001351445071700170060ustar00rootroot00000000000000tests/resources/data.env000066400000000000000000000001231351445071700156600ustar00rootroot00000000000000NGINX_HOSTNAME=localhost NGINX_WEBROOT=/var/www/project NGINX_LOGS=/var/log/nginx/ tests/resources/data.ini000066400000000000000000000001111351445071700156440ustar00rootroot00000000000000[nginx] hostname=localhost webroot=/var/www/project logs=/var/log/nginx/ tests/resources/data.json000066400000000000000000000002021351445071700160370ustar00rootroot00000000000000{ "nginx":{ "hostname": "localhost", "webroot": "/var/www/project", "logs": "/var/log/nginx/" } } tests/resources/data.yaml000066400000000000000000000001211351445071700160300ustar00rootroot00000000000000nginx: hostname: localhost webroot: /var/www/project logs: /var/log/nginx/ tests/resources/data.yml000066400000000000000000000001211351445071700156670ustar00rootroot00000000000000nginx: hostname: localhost webroot: /var/www/project logs: /var/log/nginx/ tests/resources/name.j2000066400000000000000000000000201351445071700154060ustar00rootroot00000000000000Hello {{name}}! tests/resources/nginx-env.j2000066400000000000000000000003231351445071700164050ustar00rootroot00000000000000server { listen 80; server_name {{ NGINX_HOSTNAME }}; root {{ NGINX_WEBROOT }}; index index.htm; access_log {{ NGINX_LOGS }}/http.access.log combined; error_log {{ NGINX_LOGS }}/http.error.log; } tests/resources/nginx.j2000066400000000000000000000003231351445071700156170ustar00rootroot00000000000000server { listen 80; server_name {{ nginx.hostname }}; root {{ nginx.webroot }}; index index.htm; access_log {{ nginx.logs }}/http.access.log combined; error_log {{ nginx.logs }}/http.error.log; } tox.ini000066400000000000000000000010571351445071700124030ustar00rootroot00000000000000[tox] envlist=py{27,34,35,36,37},pypy, py36-pyyaml5.1 py36-pyyaml3.13 py36-pyyaml3.12 py36-pyyaml3.11 py36-pyyaml3.10 skip_missing_interpreters=True [testenv] deps= -rrequirements-dev.txt py{27,34,35,36},pypy: -e.[yaml] py37: pyyaml py36-pyyaml5.1: pyyaml==5.1 py36-pyyaml3.13: pyyaml==3.13 py36-pyyaml3.12: pyyaml==3.12 py36-pyyaml3.11: pyyaml==3.11 py36-pyyaml3.10: pyyaml==3.10 commands= nosetests {posargs:tests/} whitelist_externals=make [testenv:dev] deps=-rrequirements-dev.txt usedevelop=True