pax_global_header00006660000000000000000000000064141740654150014521gustar00rootroot0000000000000052 comment=90dd831f8d43966a8f97d4536295082c3353f29e stone-3.3.1/000077500000000000000000000000001417406541500126555ustar00rootroot00000000000000stone-3.3.1/.arcconfig000066400000000000000000000001351417406541500146100ustar00rootroot00000000000000{ "conduit_uri": "https://tails.corp.dropbox.com/api/", "repository.callsign": "STONE" } stone-3.3.1/.coveragerc000066400000000000000000000000431417406541500147730ustar00rootroot00000000000000[run] branch = True source = stone/stone-3.3.1/.github/000077500000000000000000000000001417406541500142155ustar00rootroot00000000000000stone-3.3.1/.github/ISSUE_TEMPLATE/000077500000000000000000000000001417406541500164005ustar00rootroot00000000000000stone-3.3.1/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000012521417406541500210720ustar00rootroot00000000000000--- name: "\U0001F41B Bug report" about: Create a report to help us improve Stone title: '' labels: bug assignees: '' --- **Describe the bug** A clear and concise description of the bug. **To Reproduce** The steps to reproduce the behavior **Expected Behavior** A clear description of what you expected to happen. **Actual Behavior** A clear description of what actually happened **Screenshots** If applicable, add screenshots to help explain your problem. **Versions** * What version of the Stone are you using? * What version of the language are you using? * What platform are you using? (if applicable) **Additional context** Add any other context about the problem here.stone-3.3.1/.github/ISSUE_TEMPLATE/config.yml000066400000000000000000000000331417406541500203640ustar00rootroot00000000000000blank_issues_enabled: falsestone-3.3.1/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000012101417406541500221170ustar00rootroot00000000000000--- name: "\U0001F680 Feature Request" about: Suggest an idea for Stone title: '' labels: enhancement assignees: '' --- **Why is this feature valuable to you? Does it solve a problem you're having?** A clear and concise description of why this feature is valuable. Ex. I'm always frustrated when [...] **Describe the solution you'd like** A clear and concise description of what you want to happen. **Describe alternatives you've considered** A clear and concise description of any alternative solutions or features you've considered. (if applicable) **Additional context** Add any other context or screenshots about the feature request here. stone-3.3.1/.github/ISSUE_TEMPLATE/question_help.md000066400000000000000000000011631417406541500216020ustar00rootroot00000000000000--- name: "\U0001F4AC Questions / Help" about: Get help with issues you are experiencing title: '' labels: help-wanted, question assignees: '' --- **Before you start** Have you checked StackOverflow, previous issues, and Dropbox Developer Forums for help? **What is your question?** A clear and concise description the question. **Screenshots** If applicable, add screenshots to help explain your question. **Versions** * What version of Stone are you using? * What version of the language are you using? * What platform are you using? (if applicable) **Additional context** Add any other context about the question here.stone-3.3.1/.github/dependabot.yml000066400000000000000000000011331417406541500170430ustar00rootroot00000000000000# To get started with Dependabot version updates, you'll need to specify which # package ecosystems to update and where the package manifests are located. # Please see the documentation for all configuration options: # https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates version: 2 updates: - package-ecosystem: "pip" # See documentation for possible values directory: "/" # Location of package manifests schedule: interval: "monthly" - package-ecosystem: "github-actions" directory: "/" schedule: interval: "monthly" stone-3.3.1/.github/pull_request_template.md000066400000000000000000000007271417406541500211640ustar00rootroot00000000000000 ## **Checklist** **General Contributing** - [ ] Have you read the Code of Conduct and signed the [CLA](https://opensource.dropbox.com/cla/)? **Is This a Code Change?** - [ ] Non-code related change (markdown/git settings etc) - [ ] Code Change - [ ] Example/Test Code Change **Validation** - [ ] Have you ran `tox`? - [ ] Do the tests pass?stone-3.3.1/.github/workflows/000077500000000000000000000000001417406541500162525ustar00rootroot00000000000000stone-3.3.1/.github/workflows/CI.yaml000066400000000000000000000022101417406541500174240ustar00rootroot00000000000000name: CI on: pull_request: jobs: CI: continue-on-error: true runs-on: ${{ matrix.os }} # Supported Versions: # https://github.com/actions/python-versions/blob/main/versions-manifest.json strategy: matrix: os: [macos-latest, ubuntu-latest] python-version: [3.6, 3.7, 3.8, pypy-3.7] steps: - uses: actions/checkout@v2 - name: Setup Python environment uses: actions/setup-python@v2.2.1 with: python-version: ${{ matrix.python-version }} - name: Install Requirements run: | python -m pip install --upgrade pip pip install flake8 pylint pytest pip install -r requirements.txt pip install -r test/requirements.txt python setup.py install - name: Run Linter run: | flake8 setup.py example stone test pylint --rcfile=.pylintrc setup.py example stone test - name: Run Unit Tests run: | pytest - name: Run MyPy if: matrix.python-version != 'pypy-3.7' run: | pip install enum34 mypy typed-ast types-six ./mypy-run.sh stone-3.3.1/.github/workflows/coverage.yaml000066400000000000000000000014571417406541500207400ustar00rootroot00000000000000name: Coverage on: push: branches: - main pull_request: schedule: - cron: 0 0 * * * jobs: Coverage: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - name: Setup Python environment uses: actions/setup-python@v2.1.4 with: python-version: '3.7' - name: Install Requirements run: | python -m pip install --upgrade pip pip install coverage pytest pip install -r test/requirements.txt python setup.py install - name: Generate Unit Test Coverage run: | coverage run --rcfile=.coveragerc -m pytest coverage xml - name: Publish Coverage uses: codecov/codecov-action@v1.0.14 with: flags: unit fail_ci_if_error: truestone-3.3.1/.github/workflows/pypiupload.yaml000066400000000000000000000020711417406541500213240ustar00rootroot00000000000000# This workflows will upload a Python Package using Twine when a release is created # For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries name: Publish to PyPi on: release: types: [created] jobs: deploy: runs-on: ubuntu-latest strategy: matrix: python-version: [2.7, 3.x] steps: - uses: actions/checkout@v2 - name: Publish to PyPi uses: actions/setup-python@v2.1.4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install setuptools wheel twine - name: Build run: | python setup.py bdist_wheel - name: Build Sources if: matrix.python-version == '3.x' run: | python setup.py sdist - name: Publish env: TWINE_USERNAME: __token__ TWINE_PASSWORD: ${{ secrets.pypi_secret }} run: | twine check dist/* twine upload dist/* stone-3.3.1/.gitignore000066400000000000000000000002361417406541500146460ustar00rootroot00000000000000.coverage coverage.xml /.cache/ /.eggs/ /.idea/ /.tox/ /build/ /dist/ __pycache__/ .DS_Store parser.out *.egg *.egg-info/ *.pyc *.pyo *.swp *~ .pytest_cache/ stone-3.3.1/.pylintrc000066400000000000000000000007361417406541500145300ustar00rootroot00000000000000[MESSAGES CONTROL] disable= C, R, fixme, locally-disabled, protected-access, useless-else-on-loop, cell-var-from-loop, # https://github.com/PyCQA/pylint/issues/1934 bad-string-format-type, # bogus errors raise-missing-from, # errors thrown on github actions not locally unused-import, wildcard-import, bad-option-value, attribute-defined-outside-init, deprecated-lambda enable=useless-suppression [REPORTS] reports=n stone-3.3.1/CODE_OF_CONDUCT.md000066400000000000000000000004751417406541500154620ustar00rootroot00000000000000# Dropbox Code Of Conduct *Dropbox believes that an inclusive development environment fosters greater technical achievement. To encourage a diverse group of contributors we've adopted this code of conduct.* Please read the Official Dropbox [Code of Conduct](https://opensource.dropbox.com/coc/) before contributing.stone-3.3.1/CONTRIBUTING.md000066400000000000000000000041411417406541500151060ustar00rootroot00000000000000# Contributing to Stone We value and rely on the feedback from our community. This comes in the form of bug reports, feature requests, and general guidance. We welcome your issues and pull requests and try our hardest to be timely in both response and resolution. Please read through this document before submitting issues or pull requests to ensure we have the necessary information to help you resolve your issue. ## Filing Bug Reports You can file a bug report on the [GitHub Issues][issues] page. 1. Search through existing issues to ensure that your issue has not been reported. If it is a common issue, there is likely already an issue. 2. Please ensure you are using the latest version of Stone. While this may be a valid issue, we only will fix bugs affecting the latest version and your bug may have been fixed in a newer version. 3. Provide as much information as you can regarding the language version, Stone version, and any other relevant information about your environment so we can help resolve the issue as quickly as possible. ## Submitting Pull Requests We are more than happy to receive pull requests helping us improve the state of our SDK. You can open a new pull request on the [GitHub Pull Requests][pr] page. 1. Please ensure that you have read the [License][license], [Code of Conduct][coc] and have signed the [Contributing License Agreement (CLA)][cla]. 2. Please add tests confirming the new functionality works. Pull requests will not be merged without passing continuous integration tests unless the pull requests aims to fix existing issues with these tests. ## Testing the Code Tests live under the `test/` folder. They can be run by running the following command: ``` $ python setup.py test ``` They can also be run as a part of `tox` and they should be ran in a virtual environment to ensure isolation of the testing environment. [issues]: https://github.com/dropbox/stone/issues [pr]: https://github.com/dropbox/stone/pulls [coc]: https://github.com/dropbox/stone/blob/main/CODE_OF_CONDUCT.md [license]: https://github.com/dropbox/stone/blob/main/LICENSE [cla]: https://opensource.dropbox.com/cla/stone-3.3.1/LICENSE000066400000000000000000000021701417406541500136620ustar00rootroot00000000000000Copyright (c) 2020 Dropbox Inc., http://www.dropbox.com/ Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.stone-3.3.1/MANIFEST.in000066400000000000000000000001621417406541500144120ustar00rootroot00000000000000include ez_setup.py include LICENSE include *.rst include stone/target/swift_rsrc/* include test/requirements.txt stone-3.3.1/README.rst000066400000000000000000000156151417406541500143540ustar00rootroot00000000000000.. image:: https://cfl.dropboxstatic.com/static/images/sdk/stone_banner.png :target: https://github.com/dropbox/stone .. image:: https://img.shields.io/pypi/pyversions/stone.svg :target: https://pypi.python.org/pypi/stone .. image:: https://img.shields.io/pypi/v/stone.svg :target: https://pypi.python.org/pypi/stone .. image:: https://codecov.io/gh/dropbox/stone/branch/main/graph/badge.svg :target: https://codecov.io/gh/dropbox/stone The Official Api Spec Language for Dropbox. `Documentation`_ can be found on GitHub Installation ============ Install stone using ``pip``:: $ pip install --user stone This will install a script ``stone`` to your PATH that can be run from the command line:: $ stone -h Alternative ----------- If you choose not to install ``stone`` using the method above, you will need to ensure that you have the Python packages ``ply`` and ``six``, which can be installed through ``pip``:: $ pip install "ply>=3.4" "six>=1.3.0" "typing>=3.5.2" If the ``stone`` package is in your PYTHONPATH, you can replace ``stone`` with ``python -m stone.cli`` as follows:: $ python -m stone.cli -h If you have the ``stone`` package on your machine, but did not install it or add its location to your PYTHONPATH, you can use the following:: $ PYTHONPATH=path/to/stone python -m stone.cli -h After installation, follow one of our `Examples`_ or read the `Documentation`_. Overview ======== Define an API once in Stone. Use backends, i.e. code generators, to translate your specification into objects and functions in the programming languages of your choice. * Introduction * Motivation_ * Installation_ * `Language Reference `_ * `Builtin Backends `_ * `Managing Specs `_ * `Evolving a Spec `_ * `Backend Reference `_ * `JSON Serializer `_ * `Network Protocol `_ *Warning: The documentation is undergoing a rewrite.* .. image:: docs/overview.png Stone is made up of several components: Language -------- A language for writing API specifications, "specs" for short. Command-Line Interface ---------------------- The CLI (``stone``) generates code based on the provided specs, backend, and additional arguments. Backends -------- There are builtin backends that come with Stone: Javascript, Python, Obj-C, Swift, and Typescript. There are other backends we've written that aren't part of the Stone package because they aren't sufficiently general, and can't realistically be re-used for non-Dropbox APIs: Go and Java. Stone includes a Python interface for defining new backends based on its intermediate representation of specs. This gives you the freedom to generate to any target. JSON Serialization ------------------ Stone defines a JSON-compatible serialization scheme. Motivation ========== Stone was birthed at Dropbox at a time when it was becoming clear that API development needed to be scaled beyond a single team. The company was undergoing a large expansion in the number of product groups, and it wasn't scalable for the API team, which traditionally dealt with core file operations, to learn the intricacies of each product and build corresponding APIs. Stone's chief goal is to decentralize API development and ownership at Dropbox. To be successful, it needed to do several things: **Decouple APIs from SDKS**: Dropbox has first-party clients for our mobile apps, desktop client, and website. Each of these is implemented in a different language. Moreover, we wanted to continue providing SDKs to third-parties, over half of whom use our SDKs. It's untenable to ask product groups that build APIs to also implement these endpoints in a half-dozen different language-specific SDKs. Without decoupling, as was the case in our v1 API, the SDKs will inevitably fall behind. Our solution is to have our SDKs automatically generated. **Improve Visibility into our APIs**: These days, APIs aren't just in the domain of engineering. Product managers, product specialists, partnerships, sales, and services groups all need to have clear and accurate specifications of our APIs. After all, APIs define Dropbox's data models and functionality. Before Stone, API design documents obseleted by changes during implementation were the source of truth. **Consistency and Predictability**: Consistency ranging from documentation tense to API patterns are important for making an API predictable and therefore easier to use. We needed an easy way to make and enforce patterns. **JSON**: To make consumption easier for third parties, we wanted our data types to map to JSON. For cases where serialization efficiency (space and time) are important, you can try using msgpack (alpha support available in the Python generator). It's possible also to define your own serialization scheme, but at that point, you may consider using something like `Protobuf `_. Stone is in active use for the `Dropbox v2 API `_. Assumptions ----------- Stone makes no assumptions about the protocol layer being used to make API requests and return responses; its first use case is the Dropbox v2 API which operates over HTTP. Stone does not come with nor enforce any particular RPC framework. Stone makes some assumptions about the data types supported in target programming languages. It's assumed that there is a capacity for representing dictionaries (unordered string keys -> value), lists, numeric types, and strings. Stone assumes that a route (or API endpoint) can have its argument and result types defined without relation to each other. In other words, the type of response does not change based on the input to the endpoint. An exception to this rule is afforded for error responses. Examples ======== We provide `Examples`_ to help get you started with a lot of the basic functionality of Stone. Getting Help ============ If you find a bug, please see `CONTRIBUTING.md`_ for information on how to report it. If you need help that is not specific to Stone, please reach out to `Dropbox Support`_. License ======= Stone is distributed under the MIT license, please see `LICENSE`_ for more information. .. _logo: {logo_link} .. _repo: https://github.com/dropbox/stone .. _`Documentation`: https://github.com/dropbox/stone/tree/main/docs .. _`Examples`: https://github.com/dropbox/stone/tree/main/example/backend .. _LICENSE: https://github.com/dropbox/stone/blob/main/LICENSE .. _CONTRIBUTING.md: https://github.com/dropbox/stone/blob/main/CONTRIBUTING.md .. _`Dropbox Support`: https://www.dropbox.com/developers/contact stone-3.3.1/codecov.yml000066400000000000000000000004441417406541500150240ustar00rootroot00000000000000coverage: status: project: default: target: auto threshold: 1% base: auto flags: - unittest paths: - "stone" if_not_found: success if_ci_failed: error informational: true only_pulls: falsestone-3.3.1/docs/000077500000000000000000000000001417406541500136055ustar00rootroot00000000000000stone-3.3.1/docs/backend_ref.rst000066400000000000000000000644511417406541500165740ustar00rootroot00000000000000***************** Writing a Backend ***************** This document explains how to write your own backend. If you're simply looking to use an included backend, see `Builtin Backends `_. .. contents:: **Table of Contents** Backend convert a spec into some other markup or code. Most commonly, a backend will target a programming language and convert a spec into classes and functions. But, backends can also create markup for things like API documentation. Backends are written as Python modules that satisfy the following conditions: 1. The filename must have a ``.stoneg.py`` extension ("g" for "generator"). For example, ``example.stoneg.py``. 2. At least one class must exist in the module that extends the ``stone.backend.CodeBackend`` class and implements the abstract ``generate()`` method. Stone automatically detects subclasses and calls the ``generate()`` method. All such subclasses will be called in ASCII order. Getting Started =============== Here's a simple no-op backend:: from stone.backend import CodeBackend class ExampleBackend(CodeBackend): def generate(self, api): pass Assuming that the backend is saved in your current directory as ``example.stoneg.py`` and that our running example spec ``users.stone`` from the `Language Reference `_ is also in the current directory. you can invoke the backend with the following command:: $ stone example.stoneg.py . users.stone Generating Output Files ======================= To create an output file, use the ``output_to_relative_path()`` method. Its only argument is the path relative to the output directory, which was specified as an argument to ``stone``, where the file should be created. Here's an example backend that creates an output file for each namespace. Each file is named after a respective namespace and have a ``.cpp`` extension. Each file contains a one line C++-style comment:: from stone.backend import CodeBackend class ExampleBackend(CodeBackend): def generate(self, api): for namespace_name in api.namespaces: with self.output_to_relative_path(namespace_name + '.cpp'): self.emit('/* {} */'.format(namespace_name)) Using the API Object ==================== The ``generate`` method receives an ``api`` variable, which represents the API spec as a Python object. The object is an instance of the ``stone.api.Api`` class. From this object, you can access all the defined namespaces, data types, and routes. Api --- namespaces A map from namespace name to Namespace object. route_schema A Struct object that defines the schema for route attributes. Namespace --------- name The name of the namespace. doc The documentation string for the namespace. This is a concatenation of the docstrings for this namespace across all spec files in the order that they were specified to `stone` on the command line. The string has no leading or trailing whitespace except for a newline at the end. If no documentation string exists, this is ``None``. routes A list of Route objects in alphabetical order. route_by_name A map from route name to Route object. For routes of multiple versions, only the route at version 1 is included. This field is deprecated by ``routes_by_name`` and will be removed in the future. routes_by_name A map from route name to RoutesByVersion object containing a group of Route objects at different versions. data_types A list of user-defined DataType objects in alphabetical order. data_type_by_name A map from data type name to DataType object. aliases A list of Alias objects in alphabetical order. Aliases will only be available if the backend has set its ``preserve_aliases`` class variable to true. alias_type_by_name A map from alias name to Alias object. annotation_types A list of user-defined AnnotationType objects. annotation_type_by_name A map from annotation name to AnnotationType object. get_imported_namespaces(must_have_imported_data_type=False, consider_annotations=False, consider_annotation_types=False) A list of Namespace objects. A namespace is a member of this list if it is imported by the current namespace and a data type or alias is referenced from it. If you want only namespaces with aliases referenced, set the ``must_have_imported_data_type`` parameter to true. Namespaces are in ASCII order by name. By default, namespaces where only annotations or annotation types are referenced are not returned. To include these namespaces, set ``consider_annotations`` or ``consider_annotation_types`` parameters to true. get_namespaces_imported_by_route_io() A list of Namespace objects. A namespace is a member of this list if it is imported by the current namespace and has a data type from it referenced as an argument, result, or error of a route. Namespaces are in ASCII order by name. get_route_io_data_types() A list of all user-defined data types that are referenced as either an argument, result, or error of a route. If a List or Nullable data type is referenced, then the contained data type is returned assuming it's a user-defined type. linearize_data_types() Returns a list of all data types used in the namespace. Because the inheritance of data types can be modeled as a DAG, the list will be a linearization of the DAG. It's ideal to generate data types in this order so that user-defined types that reference other user-defined types are defined in the correct order. linearize_aliases() Returns a list of all aliases used in the namespace. The aliases are ordered to ensure that if they reference other aliases those aliases come earlier in the list. Route ----- name The name of the route. deprecated Set to a ``DeprecationInfo`` object if this route is deprecated. If the route was deprecated by a newer route, ``DeprecationInfo`` will have a ``by`` attribute populated with the new route. doc The documentation string for the route. arg_data_type A DataType object of the arg to the route. result_data_type A DataType object of the result of the route. error_data_type A DataType object of the error of the route. attrs A map from string keys to values that is a direct copy of the attrs specified in the route definition. Values are limited to Python primitives (None, bool, float, int, str) and `TagRef objects <#union-tag-reference>`_. See the Python object definition for more information. RoutesByVersion --------------- at_version A map from version number to Route object. The version number is an integer starting at 1. DataType -------- name The name of the data type. See ``stone.data_type`` for all primitive type definitions and their attributes. Struct ------ name The name of the struct. namespace The namespace the struct was defined in. doc The documentation string for the struct. fields A list of StructField objects defined by this struct. Does not include any inherited fields. all_fields A list of StructField objects including inherited fields. Required fields come before optional fields. all_required_fields A list of StructField objects required fields. Includes inherited fields. all_optional_fields A list of StructField objects for optional fields. Includes inherited fields. Optional fields are those that have defaults, or have a data type that is nullable. parent_type If it exists, it points to a DataType object (another struct) that this struct inherits from. has_documented_type_or_fields(include_inherited_fields=False) Returns whether this type, or any of its fields, are documented. Use this when deciding whether to create a block of documentation for this type. has_documented_fields(include_inherited_fields=False) Returns whether at least one field is documented. get_all_subtypes_with_tags() Unlike other enumerated-subtypes-related functionality, this method returns not just direct subtypes, but all subtypes of this struct. The tag of each subtype is the tag of the enumerated subtype from which it descended. The return value is a list of tuples representing subtypes. Each tuple has two items. First, the type tag to be used for the subtype. Second, a ``Struct`` object representing the subtype. Use this when you need to generate a lookup table for a root struct that maps a generated class representing a subtype to the tag it needs in the serialized format. Raises an error if the struct doesn't enumerate subtypes. get_enumerated_subtypes() Returns a list of subtype fields. Each field has a ``name`` attribute which is the tag for the subtype. Each field also has a ``data_type`` attribute that is a ``Struct`` object representing the subtype. Raises an error if the struct doesn't enumerate subtypes. has_enumerated_subtypes() Returns whether this struct enumerates its subtypes. is_catch_all() Indicates whether this struct should be used in the event that none of its known enumerated subtypes match a received type tag. Raises an error if the struct doesn't enumerate subtypes. is_member_of_enumerated_subtypes_tree() Returns true if this struct enumerates subtypes or if its parent does. Structs that are members of trees must be able to be serialized without their inherited fields. get_examples() Returns an `OrderedDict `_ mapping labels to ``Example`` objects. StructField ----------- name The name of the field. doc The documentation string for the field. data_type The DataType of the field. has_default Whether this field has a default if it is unset. default The default for this field. Errors if no default is defined. The Python type of the default depends on the data type of the field. The following table shows the mapping: ========================== ============ ============ Primitive Python 2.x Python 3.x ========================== ============ ============ Bytes str bytes Boolean bool bool Float{32,64} float float Int{32,64}, UInt{32,64} long int List list list String unicode str Timestamp str str ========================== ============ ============ If the data type of a field is a union, its default can be a `TagRef object <#union-tag-reference>`_. No defaults are supported for structs. Union ----- name The name of the union. namespace The namespace the struct was defined in. doc The documentation string for the union. fields A list of UnionField objects defined by this union. Does not include any inherited fields. all_fields A list of all UnionField objects that make up the union. Required fields come before optional fields. parent_type If it exists, it points to a DataType object (another union) that this union inherits from. catch_all_field A UnionField object representing the catch-all field. has_documented_type_or_fields(include_inherited_fields=False) Returns whether this type, or any of its fields, are documented. Use this when deciding whether to create a block of documentation for this type. has_documented_fields(include_inherited_fields=False) Returns whether at least one field is documented. get_examples() Returns an `OrderedDict`_ mapping labels to ``Example`` objects. UnionField ---------- name The name of the field. doc The documentation string for the field. data_type The DataType of the field. catch_all A boolean indicating whether this field is the catch-all for the union. Alias ----- name The target name. data_type The DataType referenced by the alias as the source. doc The documentation string for the alias. Example ------- label The label for the example defined in the spec. text A textual description of the example that follows the label in the spec. Is ``None`` if no text was provided. example A JSON representation of the example that is generated based on the example defined in the spec. .. _emit_methods: Emit*() Methods =============== There are several ``emit*()`` methods included in a ``CodeBackend`` that each serve a different purpose. ``emit(s='')`` Adds indentation, then the input string, and lastly a newline to the output buffer. If ``s`` is an empty string (default) then an empty line is created with no indentation. ``emit_wrapped_text(s, prefix='', initial_prefix='', subsequent_prefix='', width=80, break_long_words=False, break_on_hyphens=False)`` Adds the input string to the output buffer with indentation and wrapping. The wrapping is performed by the ``textwrap.fill`` Python library function. ``prefix`` is prepended to every line of the wrapped string. ``initial_prefix`` is prepended to the first line of the wrapped string ``subsequent_prefix`` is prepended to every line after the first. On a line, ``prefix`` will always come before ``initial_prefix`` and ``subsequent_prefix``. ``width`` is the target width of each line including indentation and prefixes. If true, ``break_long_words`` breaks words longer than width. If false, those words will not be broken, and some lines might be longer than width. If true, ``break_on_hyphens`` allows breaking hyphenated words; wrapping will occur preferably on whitespaces and right after the hyphen in compound words. ``emit_raw(s)`` Adds the input string to the output buffer. The string must end in a newline. It may contain any number of newline characters. No indentation is generated. Indentation =========== The ``stone.backend.CodeBackend`` class provides a context manager for adding incremental indentation. Here's an example:: from stone.backend import CodeBackend class ExampleBackend(CodeBackend): def generate(self, api): with self.output_to_relative_path('ex_indent.out'): with self.indent() self.emit('hello') self._output_world() def _output_world(self): with self.indent(): self.emit('world') The contents of ``ex_indent.out`` is:: hello world Indentation is always four spaces. We plan to make this customizable in the future. Helpers for Code Generation =========================== ``generate_multiline_list(items, before='', after='', delim=('(', ')'), compact=True, sep=',', skip_last_sep=False)`` Given a list of items, emits one item per line. This is convenient for function prototypes and invocations, as well as for instantiating arrays, sets, and maps in some languages. ``items`` is the list of strings that make up the list. ``before`` is the string that comes before the list of items. ``after`` is the string that follows the list of items. The first element of ``delim`` is added immediately following ``before``, and the second element is added prior to ``after``. If ``compact`` is true, the enclosing parentheses are on the same lines as the first and last list item. ``sep`` is the string that follows each list item when compact is true. If compact is false, the separator is omitted for the last item. ``skip_last_sep`` indicates whether the last line should have a trailing separator. This parameter only applies when ``compact`` is false. ``block(before='', after='', delim=('{','}'), dent=None, allman=False)`` A context manager that emits configurable lines before and after an indented block of text. This is convenient for class and function definitions in some languages. ``before`` is the string to be output in the first line which is not indented. ``after`` is the string to be output in the last line which is also not indented. The first element of ``delim`` is added immediately following ``before`` and a space. The second element is added prior to a space and then ``after``. ``dent`` is the amount to indent the block. If none, the default indentation increment is used. ``allman`` indicates whether to use ``Allman`` style indentation instead of the default ``K&R`` style. For more about indent styles see `Wikipedia `_. ``process_doc(doc, handler)`` Helper for parsing documentation `references `_ in Stone docstrings and replacing them with more suitable annotations for the target language. ``doc`` is the docstring to scan for references. ``handler`` is a function you define with the following signature: `(tag: str, value: str) -> str`. ``handler`` will be called for every reference found in the docstring with the tag and value parsed for you. The returned string will be substituted in the docstring for the reference. Backend Instance Variables ========================== logger This is an instance of the `logging.Logger `_ class from the Python standard library. Messages written to the logger will be output to standard error as the backend runs. target_folder_path The path to the output folder. Use this when the ``output_to_relative_path`` method is insufficient for your purposes. Data Type Classification Helpers ================================ ``stone.ir`` includes functions for classifying data types. These are useful when backends need to discriminate between types. The following are available:: is_binary_type(data_type) is_boolean_type(data_type) is_composite_type(data_type) is_integer_type(data_type) is_float_type(data_type) is_list_type(data_type) is_nullable_type(data_type) is_numeric_type(data_type) is_primitive_type(data_type) is_string_type(data_type) is_struct_type(data_type) is_timestamp_type(data_type) is_union_type(data_type) is_user_defined_type(data_type) is_void_type(data_type) There is also an ``unwrap_nullable(data_type)`` function that takes a ``Nullable`` object and returns the type that it wraps. If the argument is not a ``Nullable``, then it's returned unmodified. Similarly, ``unwrap_aliases(data_type)`` takes an ``Alias`` object and returns the type that it wraps. There might be multiple levels of aliases wrapping the type. The ``unwrap(data_type)`` function will return the underlying type once all wrapping ``Nullable`` and ``Alias`` objects have been removed. Note that an ``Alias`` can wrap a ``Nullable`` and a ``Nullable`` can wrap an ``Alias``. Union Tag Reference =================== Tag references can occur in two instances. First, as the default of a struct field with a union data type. Second, as the value of a route attribute. References are limited to members with void type. TagRef ------ union_data_type The Union object that is the data type of the field. tag_name The name of the union member with void type that is the field default. To check for a default value that is a ``TagRef``, use ``is_tag_ref(val)`` which can be imported from ``stone.data_type``. Command-Line Arguments ====================== Backends can receive arguments from the command-line. A ``--`` is used to separate arguments to the ``stone`` program and the backend. For example:: $ stone python_types . ../sample.stone -- -h usage: python-types-backend [-h] [-r ROUTE_METHOD] optional arguments: -h, --help show this help message and exit -r ROUTE_METHOD, --route-method ROUTE_METHOD A string used to construct the location of a Python method for a given route; use {ns} as a placeholder for namespace name and {route} for the route name. This is used to translate Stone doc references to routes to references in Python docstrings. Note: This is for backend-specific arguments which follow arguments to Stone after a "--" delimiter. The above prints the help string specific to the included Python backend. Command-line parsing relies on Python's `argparse module `_ so familiarity with it is helpful. To define a command-line parser for a backend, assign an `Argument Parser `_ object to the ``cmdline_parser`` class variable of your backend. Set the ``prog`` keyword to the name of your backend, otherwise, the help string will claim to be for ``stone``. The ``generate`` method will have access to an ``args`` instance variable with an `argparse.Namespace object `_ holding the parsed command-line arguments. Here's a minimal example:: import argparse from stone.backend import CodeBackend _cmdline_parser = argparse.ArgumentParser(prog='example') _cmdline_parser.add_argument('-v', '--verbose', action='store_true', help='Prints to stdout.') class ExampleBackend(CodeBackend): cmdline_parser = _cmdline_parser def generate(self, api): if self.args.verbose: print 'Running in verbose mode' Examples ======== The following examples can all be found in the ``stone/example/backend`` folder. Example 1: List All Namespaces ------------------------------ We'll create a backend ``ex1.stoneg.py`` that generates a file called ``ex1.out``. Each line in the file will be the name of a defined namespace:: from stone.backend import CodeBackend class ExampleBackend(CodeBackend): def generate(self, api): """Generates a file that lists each namespace.""" with self.output_to_relative_path('ex1.out'): for namespace in api.namespaces.values(): self.emit(namespace.name) We use ``output_to_relative_path()`` a member of ``CodeBackend`` to specify where the output of our ``emit*()`` calls go (See more emit_methods_). Run the backend from the root of the Stone folder using the example specs we've provided:: $ stone example/backend/ex1/ex1.stoneg.py output/ex1 example/api/dbx-core/*.stone Now examine the contents of the output:: $ cat example/backend/ex1/ex1.out files users Example 2: A Python module for each Namespace --------------------------------------------- Now we'll create a Python module for each namespace. Each module will define a ``noop()`` function:: from stone.backend import CodeBackend class ExamplePythonBackend(CodeBackend): def generate(self, api): """Generates a module for each namespace.""" for namespace in api.namespaces.values(): # One module per namespace is created. The module takes the name # of the namespace. with self.output_to_relative_path('{}.py'.format(namespace.name)): self._generate_namespace_module(namespace) def _generate_namespace_module(self, namespace): self.emit('def noop():') with self.indent(): self.emit('pass') Note how we used the ``self.indent()`` context manager to increase the indentation level by a default 4 spaces. If you want to use tabs instead, set the ``tabs_for_indents`` class variable of your extended ``CodeBackend`` class to ``True``. Run the backend from the root of the Stone folder using the example specs we've provided:: $ stone example/backend/ex2/ex2.stoneg.py output/ex2 example/api/dbx-core/*.stone Now examine the contents of the output:: $ cat output/ex2/files.py def noop(): pass $ cat output/ex2/users.py def noop(): pass Example 3: Define Python Classes for Structs -------------------------------------------- As a more advanced example, we'll define a backend that makes a Python class for each struct in our specification. We'll use some provided helpers from ``stone.backends.python``:: from stone.data_type import is_struct_type from stone.backend import CodeBackend from stone.backends.python_helpers import ( fmt_class, fmt_var, ) class ExamplePythonBackend(CodeBackend): def generate(self, api): """Generates a module for each namespace.""" for namespace in api.namespaces.values(): # One module per namespace is created. The module takes the name # of the namespace. with self.output_to_relative_path('{}.py'.format(namespace.name)): self._generate_namespace_module(namespace) def _generate_namespace_module(self, namespace): for data_type in namespace.linearize_data_types(): if not is_struct_type(data_type): # Only handle user-defined structs (avoid unions and primitives) continue # Define a class for each struct class_def = 'class {}(object):'.format(fmt_class(data_type.name)) self.emit(class_def) with self.indent(): if data_type.doc: self.emit('"""') self.emit_wrapped_text(data_type.doc) self.emit('"""') self.emit() # Define constructor to take each field args = ['self'] for field in data_type.fields: args.append(fmt_var(field.name)) self.generate_multiline_list(args, 'def __init__', ':') with self.indent(): if data_type.fields: self.emit() # Body of init should assign all init vars for field in data_type.fields: if field.doc: self.emit_wrapped_text(field.doc, '# ', '# ') member_name = fmt_var(field.name) self.emit('self.{0} = {0}'.format(member_name)) else: self.emit('pass') self.emit() stone-3.3.1/docs/builtin_backends.rst000066400000000000000000000332171417406541500176450ustar00rootroot00000000000000**************** Builtin Backends **************** Using a backend, you can convert the data types and routes in your spec into objects in your programming language of choice. Stone includes backends for an assortment of languages, including: * `Python <#python-guide>`_ * Python `Type Stubs `_ * Javascript * Objective-C * Swift * Typescript If you're looking to write your own backend, see `Backend Reference `_. We would love to see a contribution of a PHP or Ruby backend. Compile with the CLI ==================== Compiling a spec and generating code is done using the ``stone`` command-line interface (CLI):: $ stone -h usage: stone [-h] [-v] [--clean-build] [-f FILTER_BY_ROUTE_ATTR] [-w WHITELIST_NAMESPACE_ROUTES | -b BLACKLIST_NAMESPACE_ROUTES] backend output [spec [spec ...]] StoneAPI positional arguments: backend Either the name of a built-in backend or the path to a backend module. Paths to backend modules must end with a .stoneg.py extension. The following backends are built-in: js_client, js_types, tsd_client, tsd_types, python_types, python_client, swift_client output The folder to save generated files to. spec Path to API specifications. Each must have a .stone extension. If omitted or set to "-", the spec is read from stdin. Multiple namespaces can be provided over stdin by concatenating multiple specs together. optional arguments: -h, --help show this help message and exit -v, --verbose Print debugging statements. --clean-build The path to the template SDK for the target language. -f FILTER_BY_ROUTE_ATTR, --filter-by-route-attr FILTER_BY_ROUTE_ATTR Removes routes that do not match the expression. The expression must specify a route attribute on the left- hand side and a value on the right-hand side. Use quotes for strings and bytes. The only supported operators are "=" and "!=". For example, if "hide" is a route attribute, we can use this filter: "hide!=true". You can combine multiple expressions with "and"/"or" and use parentheses to enforce precedence. -w WHITELIST_NAMESPACE_ROUTES, --whitelist-namespace-routes WHITELIST_NAMESPACE_ROUTES If set, backends will only see the specified namespaces as having routes. -b BLACKLIST_NAMESPACE_ROUTES, --blacklist-namespace-routes BLACKLIST_NAMESPACE_ROUTES If set, backends will not see any routes for the specified namespaces. We'll generate code based on an ``calc.stone`` spec with the following contents:: namespace calc route eval(Expression, Result, EvalError) struct Expression "This expression is limited to a binary operation." op Operator = add left Int64 right Int64 union Operator add sub mult div Boolean "If value is true, rounds up. Otherwise, rounds down." struct Result answer Int64 union EvalError overflow Python Guide ============ This section explains how to use the pre-packaged Python backends and work with the Python classes that have been generated from a spec. There are two different Python backends: ``python_types`` and ``python_client``. The former generates Python classes for the data types defined in your spec. The latter generates a single Python class with a method per route, which is useful for building SDKs. We'll use the ``python_types`` backend:: $ stone python_types . calc.stone This runs the backend on the ``calc.stone`` spec. Its output target is ``.`` which is the current directory. A Python module is created for each declared namespace, so in this case only ``calc.py`` is created. Three additional modules are copied into the target directory. The first, ``stone_validators.py``, contains classes for validating Python values against their expected Stone types. You will not need to explicitly import this module, but the auto-generated Python classes depend on it. The second, ``stone_serializers.py``, contains a pair of ``json_encode()`` and ``json_decode()`` functions. You will need to import this module to serialize your objects. The last is ``stone_base.py`` which shouldn't be used directly. In the following sections, we'll interact with the classes generated in ``calc.py``. For simplicity, we'll assume we've opened a Python interpreter with the following shell command:: $ python -i calc.py For non-test projects, we recommend that you set the generation target to a path within a Python package, and use Python's import facility. Primitive Types --------------- The following table shows the mapping between a Stone `primitive type `_ and its corresponding type in Python. ========================== ============== ===================================== Primitive Python 2.x / 3 Notes ========================== ============== ===================================== Bytes bytes Boolean bool Float{32,64} float long type within range is converted. Int{32,64}, UInt{32,64} long List list String unicode / str str type is converted to unicode. Timestamp datetime ========================== ============== ===================================== Struct ------ For each struct in your spec, you will see a corresponding Python class of the same name. In our example, ``Expression``, ``Operator``, ``Answer``, ``EvalError``, and are Python classes. They have an attribute (getter/setter/deleter property) for each field defined in the spec. You can instantiate these classes and specify field values either in the constructor or by assigning to an attribute:: >>> expr = Expression(op=Operator.add, left=1, right=1) If you assign a value that fails validation, an exception is raised:: >>> expr.op = '+' Traceback (most recent call last) ... ValidationError: expected type Operator or subtype, got string Accessing a required field (non-optional with no default) that has not been set raises an error:: >>> res = Result() >>> res.answer Traceback (most recent call last): File "", line 1, in File "calc.py", line 221, in answer raise AttributeError("missing required field 'answer'") AttributeError: missing required field 'answer' Other characteristics: 1. Inheritance in Stone is represented as inheritance in Python. 2. If a field is nullable and was never set, ``None`` is returned. 3. If a field has a default but was never set, the default is returned. Union ----- For each union in your spec, you will see a corresponding Python class of the same name. You do not use a union class's constructor directly. To select a tag with a void type, use the class attribute of the same name:: >>> EvalError.overflow EvalError('overflow', None) To select a tag with a value, use the class method of the same name and pass in an argument to serve as the value:: >>> Operator.div(False) Operator('div', False) To write code that handles the union options, use the ``is_[tag]()`` methods. We recommend you exhaustively check all tags, or include an else clause to ensure that all possibilities are accounted for. For tags that have values, use the ``get_[tag]()`` method to access the value:: >>> # assume that op is an instance of Operator >>> if op.is_add(): ... # handle addition ... elif op.is_sub(): ... # handle subtraction ... elif op.is_mult(): ... # handle multiplication ... elif op.is_div(): ... round_up = op.get_div() ... # handle division Struct Polymorphism ------------------- As with regular structs, structs that enumerate subtypes have corresponding Python classes that behave identically to regular structs. The difference is apparent when a field has a data type that is a struct with enumerated subtypes. Expanding on our example from the language reference, assume the following spec:: struct Resource union file File folder Folder path String struct File extends Resource: size UInt64 struct Folder extends Resource: "No new fields." struct Response rsrc Resource If we instantiate ``Response``, the ``rsrc`` field can only be assigned a ``File`` or ``Folder`` object. It should not be assigned a ``Resource`` object. An exception to this is on deserialization. Because ``Resource`` is specified as a catch-all, it's possible when deserializing a ``Response`` to get a ``Resource`` object in the ``rsrc`` field. This indicates that the returned subtype was unknown because the recipient has an older spec than the sender. To handle catch-alls, you should use an else clause:: >>> print resp.rsrc.path # Guaranteed to work regardless of subtype >>> if isinstance(resp, File): ... # handle File ... elif isinstance(resp, Folder): ... # handle Folder ... else: ... # unknown subtype of Resource Route ----- Routes are represented as instances of a ``Route`` object. The generated Python module for the namespace will have a module-level variable for each route:: >>> eval Route('eval', 1, False, ...) Route attributes specified in the spec are available as a dict in the ``attrs`` member variable. Route deprecation is stored in the ``deprecated`` member variable. The name and version of a route are stored in the ``name`` and ``version`` member variables, respectively. Serialization ------------- We can use ``stone_serializers.json_encode()`` to serialize our objects to JSON:: >>> import stone_serializers >>> stone_serializers.json_encode(eval.result_type, Result(answer=10)) '{"answer": 10}' To deserialize, we can use ``json_decode``:: >>> stone_serializers.json_decode(eval.result_type, '{"answer": 10}') Result(answer=10) There's also ``json_compat_obj_encode`` and ``json_compat_obj_decode`` for converting to and from Python primitive types rather than JSON strings. Route Functions --------------- To generate functions that represent routes, use the ``python_client`` generator:: $ stone python_client . calc.stone -- -m client -c Client -t myservice ``-m`` specifies the name of the Python module to generate, in this case ``client.py``. The important contents of the file look as follows:: class Client(object): __metaclass__ = ABCMeta @abstractmethod def request(self, route, namespace, arg, arg_binary=None): pass # ------------------------------------------ # Routes in calc namespace def calc_eval(self, left, right, op=calc.Operator.add): """ :type op: :class:`myservice.calc.Operator` :type left: long :type right: long :rtype: :class:`myservice.calc.Result` :raises: :class:`.exceptions.ApiError` If this raises, ApiError will contain: :class:`myservice.calc.EvalError` """ arg = calc.Expression(left, right, op) r = self.request( calc.eval, 'calc', arg, None, ) return r ``-c`` specified the name of the abstract class to generate. Using this class, you'll likely want to inherit the class and implement the request function. For example, an API that goes over HTTP might have the following client:: import requests # use the popular HTTP library from .stone_serializers import json_decode, json_encode from .exceptions import ApiError # You must implement this class MyServiceClient(Client): def request(self, route, namespace, arg, arg_binary=None): url = 'https://api.myservice.xyz/{}/{}'.format( namespace, route.name) r = requests.get( url, headers={'Content-Type': 'application/json'}, data=json_encode(route.arg_type, arg)) if r.status_code != 200: raise ApiError(...) return json_decode(route.result_type, r.content) Note that care is taken to ensure that that the return type and exception type match those that were specified in the automatically generated documentation. Routes with Version Numbers --------------------------- There can be multiple versions of routes sharing the same name. For each route with a version numbers other than 1, the generated module-level route variable and route function have a version suffix appended in the form of ``{name}_v{version}``. For example, suppose we add a new version of route ``eval`` in ``calc.stone`` as follows:: ... route eval:2(Expression, ResultV2, EvalError) struct ResultV2 answer String ... The module-level variable for the route will be:: >>> eval_v2 Route('eval', 2, False, ...) And the corresponding route function in ``client.py`` will be ``calc_eval_v2``. stone-3.3.1/docs/evolve_spec.rst000066400000000000000000000152711417406541500166570ustar00rootroot00000000000000*************** Evolving a Spec *************** APIs are constantly evolving. In designing Stone, we sought to codify what changes are backwards incompatible, and added facilities to make maintaining compatibility easier. Background ========== The root of the problem is that when an API interface evolves, it does not evolve simultaneously for all communicating parties. This happens for a couple reasons: 1. The owner of the API does not have control over 3rd parties that have integrated their software at some point in the evolution of the interface. These integrations may never be updated making compatibility-awareness critical. 2. Even the owner of the API may roll out evolutions to their fleet of servers in stages, meaning that clusters of servers will have different understandings of the interface for windows of time. Sender-Recipient ================ When discussing interface compatibility, it's best to think in terms of a message sender and a message receiver, either of which may have a newer interface. If the sender has a newer version, we want to make sure that the recipient still understands the message it receives, and ignores the parts that it doesn't. If the recipient has a newer version, we want to make sure that it knows what to do when the sender's message is missing data. Backwards Incompatible Changes ------------------------------ * Removing a struct field * An old receiver may have application-layer dependencies on the field, which will cease to exist. * Changing the type of a struct field. * An old receiver may have application-layer dependencies on the field type. In statically typed languages deserialization will fail. * Adding a new tag to a closed union. * We expect receivers to exhaustively handle all tags. If a new tag is returned, the receiver's handler code will be insufficient. * Changing the type of a tag with a non-Void type. * Similar to the above, if a tag changes, the old receiver's handler code will break. * Changing any of the types of a route description to an incompatible one. * When changing an arg, result, or error data type for a route, you should think about it as applying a series of operations to convert the old data type to the new one. * The change in data type is backwards incompatible if any operation is backwards incompatible. Backwards Compatible Changes ---------------------------- * Adding a new route. * Changing the name of a stuct, union, or alias. * Adding a field to a struct that is optional or has a default. * If the receiver is newer, it will either set the field to the default, or mark the field as unset, which is acceptable since the field is optional. * If the sender is newer, it will send the new field. The receiver will simply ignore the field that it does not understand. * Change the type of a tag from Void to anything else. * The older receiver will ignore information associated with the new data type and continue to present a tag with no value to the application. * Adding a new tag to an open union. * The older receiver will not understand the incoming tag, and will simply set the union to its catch-all tag. The application-layer will handle this new tag through the same code path that handles the catch-all tag. Planning for Backwards Compatibility ==================================== * When defining a union that you're likely to add tags to in the future, use an open union. By default, unions are open. Stone exposes a virtual tag called "other" of void type to generators that is known as the "catch-all" tag for this purpose. If a recipient receives a tag that it isn't aware of, it will default the union to the "other" tag. Leader-Clients ============== We focused on senders and recipients because they illustrate the general case where any two parties may have different versions of a spec. However, your system may have an added layer of predictability where some party ("leader") is guaranteed to have the same or newer version of the spec than its "clients." It's important to note that a leader-clients relationship can be transient and opportunistic--it's important to decide if this relationship exists in your setup. The leader-client relationship comes up often: 1. A service that has an API is the "leader" for first-party or third-party clients in the wild that are accessing the service's data. The server will get a spec update, and clients will have to update their code to take advantage of the new spec. 2. Within a fleet of servers, you may have two clusters that communicate with each other, one of which receives scheduled updates before the other. A known leader can be stricter with what it receives from clients: * When the leader is acting as a recipient, it should reject any struct fields it is unaware of. It knows that the unknown fields are not because the client, acting as a sender, has a newer version of the spec. * Since a client acting as a recipient may have an older spec, it should retain the behavior of ignoring unknown fields. * If the leader is acting as a recipient, it should reject all unknown tags even if the union specifies a catch-all. * If the leader is acting as a recipient, any tag with type Void should have no associated value in the serialized message since it's not possible for a client to have converted the data type to something else. [TODO] There are more nuanced backwards compatible changes such as: A tag can be removed if the union is only sent from the server to a client. Will this level of detail just lead to errors in practice? Route Versioning ================ Building language facilities to ease route versioning has yet to be fully addressed. Right now, if you know you are making a backwards incompatible change, we suggest the following verbose approach: * Create a new route. * The Stone language syntax supports specifying a version number for a route. You can attach the version number to the end of the route name separated by a `:`. For example, to introduce version 2 for ``/get_account``, use the annotation ``/get_account:2``. * Copy the definition of any data types that are changing in a backwards incompatible way. For example, if the response data type is undergoing an incompatible change, duplicate the response data type, give it a new name, and make the necessary modifications. * Be sure to update the route signature to reference the new data type. Future Work =========== Building in a lint checker into the ``stone`` command-line interface that warns if a spec change is backwards incompatible based on the revision history. This assumes that the spec file is in a version-tracking system like git or hg. stone-3.3.1/docs/json_serializer.rst000066400000000000000000000115721417406541500175470ustar00rootroot00000000000000*************** JSON Serializer *************** Code generators include a JSON serializer which will convert a target language's representation of Stone data types into JSON. This document explores how Stone data types, regardless of language, are mapped to JSON. Primitive Types =============== ========================== ==================================================== Stone Primitive JSON Representation ========================== ==================================================== Boolean Boolean Bytes String: Base64-encoded Float{32,64} Number Int{32,64}, UInt{32,64} Number List Array String String Timestamp String: Encoded using strftime() based on the Timestamp's format argument. Void Null ========================== ==================================================== Struct ====== A struct is represented as a JSON object. Each specified field has a key in the object. For example:: struct Coordinate x Int64 y Int64 converts to:: { "x": 1, "y": 2 } If an optional (has a default or is nullable) field is not specified, the key should be omitted. For example, given the following spec:: struct SurveyAnswer age Int64 name String = "John Doe" address String? If ``name`` and ``address`` are unset and ``age`` is 28, then the struct serializes to:: { "age": 28 } Setting ``name`` or ``address`` to ``null`` is not a valid serialization; deserializers will raise an error. An explicit ``null`` is allowed for fields with nullable types. While it's less compact, this makes serialization easier in some languages. The previous example could therefore be represented as:: { "age": 28, "address": null } Enumerated Subtypes ------------------- A struct that enumerates subtypes serializes similarly to a regular struct, but includes a ``.tag`` key to distinguish the type. Here's an example to demonstrate:: struct A union* b B c C w Int64 struct B extends A x Int64 struct C extends A y Int64 Serializing ``A`` when it contains a struct ``B`` (with values of ``1`` for each field) appears as:: { ".tag": "b", "w": 1, "x": 1 } If the recipient receives a tag it cannot match to a type, it should fallback to the parent type if it's specified as a catch-all. For example:: { ".tag": "d", "w": 1, "z": 1 } Because ``d`` is unknown, the recipient checks that struct ``A`` is a catch-all. Since it is, it deserializes the message to an ``A`` object. Union ===== Similar to an enumerated subtype struct, recipients should check the ``.tag`` key to determine the union variant. Let's use the following example to illustrate how a union is serialized based on the selected variant:: union U singularity number Int64 coord Coordinate? infinity Infinity struct Coordinate x Int64 y Int64 union Infinity positive negative The serialization of ``U`` with tag ``singularity`` is:: { ".tag": "singularity" } For a union member of primitive type (``number`` in the example), the serialization is as follows:: { ".tag": "number", "number": 42 } Note that ``number`` is used as the value for ``.tag`` and as a key to hold the value. This same pattern is used for union members with types that are other unions or structs with enumerated subtypes. Union members that are ordinary structs (``coord`` in the example) serialize as the struct with the addition of a ``.tag`` key. For example, the serialization of ``Coordinate`` is:: { "x": 1, "y": 2 } The serialization of ``U`` with tag ``coord`` is:: { ".tag": "coord", "x": 1, "y": 2 } The serialization of ``U`` with tag ``infinity`` is nested:: { ".tag": "infinity", "infinity": { ".tag": "positive" } } The same rule applies for members that are enumerated subtypes. Nullable -------- Note that ``coord`` references a nullable type. If it's unset, then the serialization only includes the tag:: { ".tag": "coord" } You may notice that if ``Coordinate`` was defined to have no fields, it is impossible to differentiate between an unset value and a value of coordinate. In these cases, we prescribe that the deserializer should return a null or unset value. Compact Form ------------ Deserializers should support an additional representation of void union members: the tag itself as a string. For example, tag ``singularity`` could be serialized as simply:: "singularity" This is convenient for humans manually entering the argument, allowing them to avoid typing an extra layer of JSON object nesting. stone-3.3.1/docs/lang_ref.rst000066400000000000000000001022661417406541500161230ustar00rootroot00000000000000****************** Language Reference ****************** .. contents:: Table of Contents Your API is described by specification files written in the Stone language. Here we'll cover the various capabilities at your disposal for expressing the intricacies of your API. Choosing a Filename =================== All specification files end with a ``.stone`` extension. We recommend that the name of the file be the same as the `namespace`_ defined in the spec. If multiple files are part of the same namespace, we recommend that they all share the same prefix: the namespace name followed by an underscore. Comments ======== Any text between a hash ``#`` and a newline is considered a comment. Comments can occupy an entire line or be added after non-comments on a line. Use comments to explain a section of the spec to a reader of the spec. Unlike `documentation strings <#documentation>`_, comments are not accessible to generators and will not appear in generated output. Namespace ========= Specs must begin with a namespace declaration as is the case here:: namespace example This logically groups all of the routes and data types in the spec file into the ``example`` namespace. A spec file must declare exactly one namespace, but multiple spec files may contribute to the same namespace. Namespaces are useful for grouping related functionality together. For example, the Dropbox API has a namespace devoted to all file operations (uploading, downloading, ...), and another namespace for all operations relevant to user accounts. Basic Types =========== In the example, ``String`` and ``Timestamp`` are basic types. Here's a table of all such types and the arguments they take: ======================= ================================= ===================== Type Arguments (**bold** are required Notes and positional) ======================= ================================= ===================== Bytes An array of bytes. Boolean Float{32,64} * min_value * max_value Int{32,64}, UInt{32,64} * min_value * max_value List * **data_type**: A primitive or Lists are homogeneous. composite type. * min_items * max_items Map * **key_data_type**: Must be an Maps must have keys instance of the String base that are a String type. type. Values can be * **value_data_type**: A any primitive or primitive or composite type. composite as long as they are homogeneous. String * min_length A unicode string. * max_length * pattern: A regular expression to be used for validation. Timestamp * **format**: Specified as a This is used by the string understood by JSON-serializer since strptime(). it has no native timestamp data type. Void ======================= ================================= ===================== Positional arguments (bold in the above table) are always required and appear at the beginning of an argument list:: List(Int64) Keyword arguments are optional and are preceded by the argument name and an ``=``:: Int64(max_value=130) If both are needed, positional come before keyword arguments:: List(Int64, max_items=5) If no arguments are needed, the parentheses can be omitted:: UInt64 We'll put these to use in the user-defined types section. Mapping to Target Languages --------------------------- Code generators map the primitive types of Stone to types in a target language. For more information, consult the appropriate guide in `Builtin Backends `_. Alias ===== Aliases let you parameterize a type once with a name and optional documentation string, and then use that name elsewhere:: alias Age = UInt64(max_value=120) "The age of a human." Aliases reduce repetition, improve readability of specs, and make refactoring easier since there's a single source of truth. Aliases can reference user-defined types and other aliases, and can make a type nullable. Struct ====== A struct is a user-defined composite type made up of fields:: struct Person "Describes a member of society." name String "Given name followed by surname." age UInt64 "The number of years, rounded down." A struct can be documented by specifying a string immediately following the struct declaration. The string can be multiple lines, as long as each subsequent line is at least at the indentation of the starting quote. Refer to `Documentation`_ for more. Following the documentation is a list of fields. Fields are formatted with the field name first followed by the field type. Documentation for a field is specified on a new indented line. Inheritance ----------- Using the ``extends`` keyword, a struct can declare a parent type. The sub type inherits all of the fields of the parent:: struct ModernPerson extends Person email String(pattern="^[^@]+@[^@]+\.[^@]+$")? "Set if this person has an e-mail address." ``ModernPerson`` inherits ``name`` and ``age`` from ``Person``. Unless explicitly mentioned, generators will translate this relationship into their target language. Composition ----------- User-defined types can be composed of other user-defined types:: struct Person "Describes a member of society." name Name age UInt64 "The number of years, rounded down." food_pref FoodPreference struct Name "Separates a name into components." given_name "Also known as first name." surname "Also known as family name." union FoodPreference anything vegetarian vegan pescetarian carnivore Nested Definitions ------------------ If you're composing a type that won't be used anywhere else, you can define the type inline:: struct Person "Describes a member of society." name Name struct "Separates a name into components." given_name "Also known as first name." surname "Also known as family name." age UInt64 "The number of years, rounded down." food_pref FoodPreference union anything vegetarian vegan pescetarian carnivore The inline definition is identical to a top-level definition, except that the name is omitted as it's already specified as the type for the field. Defaults -------- A field with a primitive type can have a default set with a ``=`` followed by a value at the end of the field declaration:: struct Person name String = "John Doe" Setting a default means that the field is optional. If it isn't specified, then the field assumes the value of the default. A default cannot be set for a nullable type. Nullable types implicitly have a default of ``null``. A default can be set for a field with a union data type, but only to a union member with a void type:: struct Person "Describes a member of society." name Name age UInt64 "The number of years, rounded down." food_pref FoodPreference = anything In practice, defaults are useful when `evolving a spec `_. Examples -------- Examples let you include realistic samples of data in definitions. This gives spec readers a concrete idea of what typical values will look like. Also, examples help demonstrate how distinct fields might interact with each other. Backends have access to examples, which is useful when automatically generating documentation. An example is declared by using the ``example`` keyword followed by a label. By convention, "default" should be used as the label name for an example that can be considered a good representation of the general case for the type:: struct Person "Describes a member of society." name Name age UInt64 "The number of years, rounded down." food_pref FoodPreference = anything example boy name = male_name age = 13 example grandpa "A grandpa who has gone vegetarian." name = male_name age = 93 food_pref = vegetarian struct Name "Separates a name into components." given_name "Also known as first name." surname "Also known as family name." example male_name given_name = "Greg" surname = "Kurtz" Every required field (not nullable and no default) must be specified. ``null`` can be used to mark that a nullable type is not present. An optional multi-line documentation string can be specified after the line declaring the example and before the example fields. Note that when you have a set of nested types, each type defines examples for its fields with primitive types. For fields with user-defined types, the value of the example must be a label of an example in the target type. Lists can be expressed with brackets:: struct ShoppingList items List(String) example default items = ["bananas", "yogurt", "cheerios"] Maps are expressed with curly braces:: struct Colors similar_colors Map(String, List(String)) example default similar_colors = {"blue": ["aqua", "azure"], "red": ["crimson"], "green": []} Map examples can also be multiline (Be mindful of indentation rules):: struct Digits digit_mapping Map(String, Map(String, Int32)) example default digit_mapping = { "one": { "one": 11, "two": 12 }, "two": { "one": 21, "two": 22 } } Union ===== Stone's unions are `tagged unions `_. Think of them as a type that can store one of several different possibilities at a time. Each possibility has an identifier that is called a "tag". Each tag is associated with a type (``inactive`` stores a ``Timestamp``). If the type is omitted as in the case of ``active``, the type is implicitly ``Void``. In this example, the union ``Shape`` has tags ``point``, ``square``, and ``circle``:: union Shape point square Float64 "The value is the length of a side." circle Float64 "The value is the radius." The primary advantage of a union is its logical expressiveness. You'll often encounter types that are best described as choosing between a set of options. Avoid the common anti-pattern of using a struct with a nullable field for each option, and relying on your application logic to enforce that only one is set. Another advantage is that for languages that support tagged unions (Swift is a recent adopter), the compiler can check that the application code handles all possible cases and that accesses are safe. Backends will take advantage of such features when they are available in the target language. Like a struct, a documentation string can follow the union declaration and/or follow each tag definition. Closed Unions ------------- By default, unions are open. That is, for the sake of backwards compatibility, a recipient of a message should be prepared to handle a tag that wasn't defined in the version of the API specification known to it. Stone exposes a virtual tag called ``other`` of void type to generators that is known as the "catch-all" tag for this purpose. If a recipient receives a tag that it isn't aware of, it will default the union to the ``other`` tag. If you don't need this flexibility, and can promise that no additional tags will be added in the future, you can "close" the union. To do so, use the ``union_closed`` keyword:: union_closed Resource file folder With the above specification, a recipient can confidently handle the "file" and "folder" tags and trust that no other value will ever be encountered. Note: We defaulted unions to being open because it's preferable for a specification writer to forget to close a union than forget to open one. The latter case is backwards-incompatible change for clients. .. _union-inheritance: Inheritance ----------- Using the ``extends`` keyword, a union can declare a parent type. The new union inherits all of the options of the parent type. However, this relationship is not expected to be translated by generators into most target languages. The reason for this is that unlike struct inheritance, union inheritance allows the parent type to substitute the child type rather than the reverse. That's because the selected tag will always be known by the child type, but a child's tag won't necessarily be known by the parent. In most languages, this relationship cannot be natively modeled. .. _union-examples: Examples -------- Examples for unions must only specify one field, since only one union member can be selected at a time. For example:: union Shape point square Float64 "The value is the length of a side." circle Float64 "The value is the radius." example default point = null example big_circle circle = 1024.0 In the ``default`` example, note that tags with void types are specified with a value of ``null``. In the ``big circle`` example, the ``circle`` tag has an associated float value. Struct Polymorphism =================== If a struct enumerates its subtypes, an instance of any subtype will satisfy the type constraint. This is useful when wanting to discriminate amongst types that are part of the same hierarchy while simultaneously being able to avoid discriminating when accessing common fields. To declare the enumeration, define a union following the documentation string of the struct if one exists. Unlike a regular union, it is unnamed. Each member of the union specifies a tag followed by the name of a subtype. The tag (known as the "type tag") is present in the serialized format to distinguish between subtypes. For example:: struct Resource union file File folder Folder path String struct File extends Resource ... struct Folder extends Resource ... Anywhere ``Resource`` is referenced, an instance of ``File`` or ``Folder`` satisfies the type constraint. A struct that enumerates subtypes cannot inherit from any other struct. Also, type tags cannot match any field names. Open vs. Closed --------------- Similar to a union, a struct with enumerated types defaults to open but can be explicitly marked as closed:: struct Resource "Sample doc." union_closed file File folder Folder path String struct File extends Resource: ... struct Folder extends Resource: ... If recipient receives a tag for a subtype that it is unaware of, it will substitute the base struct in its place. In the example above, if the subtype is a ``Symlink`` (not shown), then the recipient will return a ``Resource`` in its place. Nullable Type ============= When a type is followed by a ``?``, the type is nullable:: String? Nullable means that the type can be unspecified, ie. ``null``. Code generators should use a language's native facilities for null, `boxed types `_, and `option types `_ if possible. For languages that do not support these features, a separate function to check for the presence of a type is the preferred method. A nullable type is considered optional. If it is not specified, it assumes the value of null. Route ===== Routes correspond to your API endpoints. Each route is defined by a signature of three data types formatted as ``(Arg, Result, Error)``. Here's an example:: namespace calc route binary_op(BinaryOpArg, Result, BinaryOpError) "Performs the requested binary operation calculation." struct BinaryOpArg op Operator left Int64 right Int64 union Operator add sub struct Result answer Int64 union BinaryOpError overflow The route is named ``binary_op``. ``BinaryOpsArg`` is the argument to the route. ``Result`` is returned on success. ``BinaryOpError`` is returned on failure. As is the case with structs and unions, a documentation string may follow the route signature. Attributes ---------- A full description of an API route tends to require vocabulary that is specific to a service. For example, the Dropbox API needs a way to specify different hostnames that routes map to, and a way to indicate which routes need authentication. To cover this open-ended use case, routes can have a set of custom attributes (``key = value`` pairs) like follows:: route r(Void, Void, Void) attrs key1 = "value1" key2 = 1234 key3 = false These attributes are defined and typed in a special struct named ``Route`` that must be defined in the ``stone_cfg`` namespace. This is a special namespace that isn't exposed to generators:: namespace stone_cfg struct Route key1 String key2 Int64 key3 Boolean key4 String = "hello" As you can see, ``key4`` can be omitted from the attrs of route ``r`` because it has a default. A value can reference a union tag with void type:: namespace sample route r(Void, Void, Void) attrs key = a union U a b Route schema:: namespace stone_cfg import sample struct Route key sample.U Deprecation ----------- You can mark a route as deprecated as follows:: route binary_op(Arg, Void, Void) deprecated If the route is deprecated in favor of a newer route, use ``deprecated by`` followed by the new route's name:: route binary_op(BinaryOpArg, Result, BinaryOpError) deprecated by binary_op_v2 route binary_op_v2((BinaryOpArg, ResultV2, BinaryOpError)) The new route ``binary_op_v2`` happens to use the same argument and error types, but its result type has changed. Versioning ---------- It's possible to have multiple versions of the same route. You can do so by adding an optional version number to the end of the route name separated by a colon (``:``). The version number needs to be a positive integer. When no version number is specified, the default value is 1, as shown below:: route get_metadata:2(Void, Void, Void) The version number can also be specified in the same way when deprecating a route using ``deprecated by``:: route get_metadata(Void, Void, Void) deprecated by get_metadata:2 Import ====== You can refer to types and aliases in other namespaces by using the ``import`` directive. For example, we can define all of of our calculator types in a ``common`` namespace in ``common.stone``:: namespace common struct BinaryOpArg op Operator left Int64 right Int64 union Operator add sub struct Result answer Int64 union BinaryOpError overflow Now in ``calc.stone``, we can import all of these types and define the route:: namespace calc import common route binary_op(common.BinaryOpArg, common.Result, common.BinaryOpError) "Performs the requested binary operation calculation." When referencing data types in ``common``, use the prefix ``common.``. For example, ``common.AccountId`` and ``common.BasicAccount``. Two namespaces cannot import each other. This is known as a circular import and is prohibited to make generating languages like Python possible. .. _doc: Patch ====== You can split the definition of a struct or union across multiple files using the ``patch`` keyword. For example, we can define ``Person`` across two different files, starting with ``public/people.stone``:: namespace people struct Person "Describes a member of society." name String "Given name followed by surname." Now in ``private/people.stone``, we can define additional fields:: namespace people patch struct Person age UInt64 "The number of years, rounded down." Only data types that have been fully-defined elsewhere can be patched. Note that patching can only be used to add additional fields, not mutate existing fields. Patching can inject both required and optional fields. For required fields, it is necessary to inject corresponding examples as well. ``public/people.stone``:: namespace people struct Person "Describes a member of society." name String "Given name followed by surname." example default name = "Stephen Cobbe" example child name = "Ken Elkabany" example adult name = "Qiming Yuan" ``private/people.stone``:: namespace people patch struct Person age UInt64 "The number of years, rounded down." example default age = 30 example child name = 12 example adult name = 45 .. _doc: Annotations ====== Annotations are special decorator tags that can be applied to fields in a Stone spec. Built-in annotations correspond to actions that Stone will perform on the field, and custom annotations can be created to mark fields that require special processing in client code. Annotations can be stacked on top of one another in most cases. Currently, Stone supports the following annotations: Omission ---------- Omission is the server-side notion of changing the API interface depending on the caller. "Omitted" annotations are annotations that associate a field with a particular set of caller permissions. "Caller permissions" are simply a list of raw string tags that the server determines apply to a particular caller. If the value of the Omitted annotation for a particular field is contained within the caller permissions list that the server passes to Stone at serialization time, the nullability of the field will be enforced. If not, then the field's nullability is ignored, and it will be stripped out at serialization time. This is useful in the case of maintaining a public/private interface for your API endpoints. Omitted annotations help to reduce server code redundancies and complicated public/private Stone object hierarchies. From the client's perspective, there is only one interface, be it public, private or any other arbitrary caller type that is defined in the Stone spec. It is the server's job to manage these different interfaces, depending on caller type. ``public/people.stone``:: namespace people struct Person "Describes a member of society." name String "Given name followed by surname." example default name = "Stephen Cobbe" ``private/people.stone``:: namespace people annotation InternalOnly = Omitted("internal") patch struct Person sensitive_id UInt64 @InternalOnly "A sensitive ID that should not be revealed publicly." example default sensitive_id = 1234 In this example, the field `sensitive_id` will only be returned for callers that have the "internal" permission in the permissions list that the server passes into Stone at serialization time. This helps to streamline server logic. Endpoint handlers can simply compute the full public/private super-type, and then rely on the serialization layer to strip out the appropriate fields, depending on the caller type. For expensive fields, endpoint handler logic can be forked based on caller type with the understanding that nullability will be selectively enforced, depending on caller type. Note: as a simplifying assumption, fields can be tagged with at most one caller type. Redaction ---------- Redaction is the act of removing sensitive data during serialization for the purpose of logging. "Redacted" annotations are annotations that associate a field with a particular type of redaction, either blotting out (e.g "***") or hashing. The redacting action is performed during serialization in the context of logging. This keeps sensitive information outside of logs. Currently, only string and numeric typed fields are eligible for redaction. Redacted annotations accept an optional regular expression string which selectively applies the redacting action to the part of the value to be redacted. If no regex is supplied, the entire value is redacted. In general, redaction is done at the field level. Aliases, however, can be marked at their definition with a redactor tag. In this case, any field of that alias type will be redacted, so redaction will be done at the type level. :: namespace people annotation NameRedactor = RedactedBlot("test_regex") annotation IdRedactor = RedactedHash() alias Name = String @NameRedactor struct Person "Describes a member of society." name Name "Given name followed by surname." sensitive_id UInt64 @IdRedactor "A sensitive ID that should not be revealed publicly." example default name = "Stephen Cobbe" Deprecation ---------- Deprecation here is the act of marking a field as deprecated (as opposted to marking a route as deprecated). Deprecated fields have special warnings injected into their documentation, and can be used to generate compile-time warnings if the field is referenced. :: namespace people annotation Deprecated = Deprecated() struct Person "Describes a member of society." name String @Deprecated "Given name followed by surname." example default name = "Stephen Cobbe" Previewing ---------- Previewing here is the act of marking a field as in preview-mode (as opposted to marking a route as in preview-mode). Preview fields have special warnings injected into their documentation, and can be used to generate compile-time warnings if the field is referenced. :: namespace people annotation Preview = Preview() struct Person "Describes a member of society." name String @Preview "Given name followed by surname." example default name = "Stephen Cobbe" Custom annotations ---------- **Note:** only the `python_types` backend supports custom annotations at this time. A custom annotation type, possibly taking some arguments, can be defined similarly to structs, and then applied the same way built-in annotations are. Note that the parameters can only be primitives (possibly nullable). Arguments can be provided as either all positional or all keyword arguments, but not a mix of both. Keyword arguments are recommended to avoid depending on the order fields are listed in the custom annotation definition. :: namespace custom_annotation_demo annotation_type Noteworthy "Describes a field with noteworthy information" importance String = "low" "The level of importance for this field (one of 'low', 'med', 'high')." annotation KindaNoteworthy = Noteworthy() annotation MediumNoteworthy = Noteworthy("med") annotation ReallyNoteworthy = Noteworthy(importance="high") alias ImportantString = String @ReallyNoteworthy struct Secrets small_secret String @KindaNoteworthy lots_of_big_ones List(ImportantString) In client code, you can access every field of a struct marked with a certain custom annotation by calling ``._process_custom_annotations(custom_annotation, field_path, processor)`` on the struct. ``processor`` will then be called with three parameters: (1) an instance of the annotation type with all the parameters populated, (2) a string denoting the path to the field being evaluated (i.e., for debugging purposes), and (3) the value of the field. The value of the field will then be replaced with the return value of ``processor``. Note that this will also affect annotated fields that are located arbitrarily deep in the struct. In the example above, if ``secret`` is a struct of type ``Secrets``, then calling ``secret._process_custom_annotations(Noteworthy, "Secrets", processor)`` will result in ``processor`` being called once as ``processor(Noteworthy("low"), "Secrets.small_secret", secret.small_secret)`` and once as ``processor(Noteworthy("high"), "Secrets.lots_of_big_ones[i]", x)`` for each element ``x`` at index ``i`` of ``secret.lots_of_big_ones``. .. _doc: Documentation ============= Documentation strings are an important part of specifications, which is why they can be attached to routes, structs, struct fields, unions, and union options. It's expected that most elements should be documented. It's not required only because some definitions are self-explanatory or adding documentation would be redundant, as is often the case when a struct field (with a doc) references a struct (with a doc). Documentation is accessible to generators. Code generators will inject documentation into the language objects that represent routes, structs, and unions. Backends for API documentation will find documentation strings especially useful. .. _doc-refs: References ---------- References help generators tailor documentation strings for a target programming language. References have the following format:: :tag:`value` Supported tags are ``route``, ``type``, ``field``, ``link``, and ``val``. route A reference to a route. The value should be the name of the route. Code generators should reference the class or function that represents the route. type A reference to a user-defined data type (Struct or Union). The value should be the name of the user-defined type. field A reference to a field of a struct or a tag of a union. If the field being referenced is a member of a different type than the docstring, then use the format `TypeName.field_name`. Otherwise, use just the field name as the value. link A hyperlink. The format of the value is `` ``, e.g. ``Stone Repo https://github.com/dropbox/stone``. Everything after the last space is considered the URI. The rest is treated as the title. For this reason, you should ensure that your URIs are `percent encoded `_. Backends should convert this to a hyperlink understood by the target language. val A value. Supported values include ``null``, ``true``, ``false``, integers, floats, and strings. Backends should convert the value to the native representation of the value for the target language. Line Continuations ================== Implicit line continuations are supported for expressions in between parentheses as long as they are at an additional indentation. For example:: route binary_op( BinaryOpArg, Result, BinaryOpError) Grammar ======= Specification:: Spec ::= Namespace Import* Definition* Namespace ::= 'namespace' Identifier Import ::= 'import' Identifier Definition ::= Alias | Route | Struct | Union Alias ::= 'alias' Identifier '=' TypeRef (NL INDENT Doc DEDENT)? Struct:: Struct ::= 'struct' Identifier Inheritance? NL INDENT Doc? Subtypes? Field* Example* DEDENT Inheritance ::= 'extends' Identifier SubtypeField ::= Identifier TypeRef NL Subtypes ::= 'union' NL INDENT SubtypeField+ DEDENT Default ::= '=' Literal Field ::= Identifier TypeRef Default? (NL INDENT Doc DEDENT)? Union:: Union ::= 'union' Identifier NL INDENT (VoidTag|Tag)* DEDENT VoidTag ::= Identifier '*'? (NL INDENT Doc DEDENT)? Tag ::= Identifier TypeRef (NL INDENT Doc DEDENT)? Route:: Route ::= 'route' Identifier (':' VersionNumber)? '(' TypeRef ',' TypeRef ',' TypeRef ')' (NL INDENT Doc DEDENT)? Type Reference:: Attributes ::= '(' (Identifier '=' (Literal | Identifier) ','?)* ')' TypeRef ::= Identifier Attributes? '?'? Primitives:: Primitive ::= 'Bytes' | 'Boolean' | 'Float32' | 'Float64' | 'Int32' | 'Int64' | 'UInt32' | 'UInt64' | 'String' | 'Timestamp' Composites:: Composite ::= 'List' Basic:: Identifier ::= (Letter | '_')? (Letter | Digit | '_')* # Should we allow trailing underscores? Letter ::= ['A'-'z'] Digit ::= ['0'-'9'] Literal :: = BoolLiteral | FloatLiteral | IntLiteral | StringLiteral BoolLiteral ::= 'true' | 'false' FloatLiteral ::= '-'? Digit* ('.' Digit+)? ('E' IntLiteral)? IntLiteral ::= '-'? Digit+ StringLiteral ::= '"' .* '"' # Not accurate VersionNumber ::= ['1'-'9'] Digit* Doc ::= StringLiteral # Not accurate NL = Newline INDENT = Incremental indentation DEDENT = Decremented indentation TODO: Need to add additional information about handling of NL, INDENT, DEDENT, and whitespace between tokens. Also, the attrs section of Routes and examples (+ lists). stone-3.3.1/docs/managing_specs.rst000066400000000000000000000036141417406541500173210ustar00rootroot00000000000000************** Managing Specs ************** Here we cover several strategies for dealing with larger projects composed of many specs, routes, and user-defined types. Using Namespaces ================ Whenever possible, group related routes and their associated data types into namespaces. This organizes your API into logical groups. Code backends should translate your namespaces into logical groups in the target language. For example, the Python backend creates a separate Python module for each namespace. Splitting a Namespace Across Files ================================== If a spec is growing large and unwieldy with thousands of lines, it might make sense to split the namespace across multiple spec files. All you need to do is create multiple ``.stone`` files with the same `Namespace `_ definition. Code backends cannot distinguish between spec files--only namespaces--so no code will be affected. When splitting a namespace across multiple spec files, each file should use the namespace name as a prefix of its filename. The ``stone`` command-line interface makes it easy to specify multiple specs:: $ stone python spec1.stone spec2.stone spec3.stone output/ $ stone python *.stone output/ Separating Public and Private Routes ==================================== Most services have a set of public routes that they publish for external developers, as well as a set of private routes that are intended to only be used internally by first-party apps. To use Stone for both public and private routes, we recommend splitting specs into ``{namespace}_public.stone`` and ``{namespace}_private.stone`` files. You may choose to simply use ``{namespace}.stone`` to represent the public spec. When publishing your API to third-party developers, you can simply include the public spec file. When generating code for internal use, you can use both the public and private spec files. stone-3.3.1/docs/network_protocol.rst000066400000000000000000000041041417406541500177500ustar00rootroot00000000000000**************** Network Protocol **************** While Stone does not standardize the network protocol used to send messages between hosts, we do anticipate that HTTP will be a popular medium. This document describes how serialized Stone objects can be transmitted using HTTP based on experiences from implementing the Dropbox v2 API. Remote Procedure Calls ====================== The majority of the routes in the Dropbox API are expected to be used by non-browser clients. Clients tend to be one of the several SDKs that Dropbox maintains for popular programming languages. Because of this, it was convenient to use the body of the HTTP request and response to store JSON-serialized Stone objects. Both requests and responses set the ``Content-Type`` header to ``application/json``. Based on the ``get_account`` route defined in the running example in the `Language Reference `_, the following is a sample exchange between a client and server. An example request:: POST /users/get_account Content-Type: application/json { "account_id": "id-48sa2f0" } An example sucessful response:: 200 OK Content-Type: application/json { "account_id": "id-48sa2f0", "email": "alex@example.org", "name": "Alexander the Great" } An example error response:: 409 Conflict Content-Type: application/json { "reason": "no_account", } HTTP Endpoint The ``get_account`` route defined in the ``users`` namespace was mapped to the url ``/users/get_account``. No host information is shown intentionally. HTTP Status Code A response indicates that its body contains an object conforming to a route's error data type by returning an HTTP 409 error code. It was important to use a "protocol layer" feature to indicate if an error had occurred. Authentication There is no authentication scheme shown in the above example. One possibility is to use an ``Authorization`` header. Browser Compatibility ===================== [TODO] Encoding JSON in headers for upload and download-style routes. stone-3.3.1/docs/overview.png000066400000000000000000001473331417406541500161740ustar00rootroot00000000000000‰PNG  IHDRŽö¾jbKGDÿÿÿ ½§“ pHYs  šœtIMEá 6ˆþê˜ IDATxÚìÝTSwž?þ'rÕ«\Û´¸“nã4NãLœÆ)ãi,a_銺à¨CáˆGzŠÓxÀo銧xŠ[uÄWÊ_a…SýÓø)NpÁ…Ì—ô -éNÒ5;f>¦5Sb‰MìûûÍ]" TùÍëqNΑäþˆ¯÷½¹÷uß¿"@™ ¤Ô¾õÔ#<òžçÞ¾}ûïoß¾ýÄ­[·õûý<…‰BÇó¼Îœ9þo¾ùæ›yóæý½3fÌèøüóÏÿß[·nuø€›"EÈÉ 2i)AØ,Â?}ùå—?øÉO~”Ëåœ\.Ÿ%“É “ÉðôÓO#ôo‰DB#„!x½^x½^€ÓéÄ­[·ÐÝÝîîn|ôÑG½üã9žç¿˜9s¦ÕívŸP ÀG‘#„’B¦2Ý‚ ^ùꫯRgÏžýhrr2÷òË/GÆÄÄ€ã8Š!„Œ"›Í†+W®à÷¿ÿ½¿¡¡!â‘Giüë_ÿzÀ%~Š!”|2%ð<ÿKžç‹¾÷½ïÍ}å•W¢M!„qâñxpá˜Íæ¯ÚÚÚ‚wîÜùµßï?ÀKÑ!„’B&«5=öØQ•J%û—ù—9:Ž"B!LGGŠŠŠüÕÕÕà8î”Ïçû¿) !„’B&T*=%³œ“@!„ ÎívãÍ7ß¼}öìÙÛ_ýµ)œ¤ÈBÉ!•L*•»sçÎÿõÎ;ïðiiiÔ—ƒB&›Í†-[¶øººº®õôôü€Š !}fP™04QQQýò—¿üÇ¿üå/ü–-[(ñ „Éøc®ÑàOú“ð›ßüæ‡sæÌi°•¢BHªù d˜3gÎ?Íœ9ówÿöoÿEM¬!dêèîîÆ?þã?öüå/¹zëÖ­W@Ãó’iŽj>g‚ ì}ä‘GN]¹r…B™b”J%>þøãy)))/Î;· €Œ¢B¦³H !ã—w<öØc~øæüÇüÇ\¥RI!„)hÆŒHHH˜„–––Ÿ E†PòA³ÄcÞ¼yŒ_ZSS3ç‘G¡ˆBȧ×ëgΟ?ÎåË—7ƒÁ‹”€J>!c"**ªöå—_þÉéÓ§ç̘A­ !dºxþùç¹ùóçGþáH  > „’BÈ('ûÔjõú .DQâA!Ó3ùú믹–––ŸÓ¾¦¨J>!£a£T*ÝÓÐÐ ‚@Ñ „i*66vÖ§Ÿ~e·Û—ƒÁÿ‡"B¦ j—±=oÞ<ëþç jµš¢A!Ó\0„J¥úÛ§Ÿ~š àE„LÔ惱!›;w®åìÙ³”xBp‹Åòèœ9sö ‹¡äƒ22æÎûo¯¿þú¼µk×R0!„ˆ”J%~ó›ß̉ŠŠªÀQDÈTGÍ®}ú'Ÿ|òüÿ÷?Êqt]!„2ÐOúÓ/ÛÚÚ~à$EƒLeTóAÈ(›3gÎo:$PâA!d(¿ûÝï™3gN1 EƒLe4Ú!£+å?øÁÆcÇŽÍ¥PBŠL&çŸ~z§³³ó±;wî¼G!S5»"dôpsæÌqüþ÷¿—ëõzŠ!„{r»ÝP(½·oßþ;^Š™Š¨Ù!£gë²eËæPâA!d8d2ǽJÑ SÕ|2:ø9sæ\ûãÿ(¥¡u !„ WGG–/_þ·¯¾úJÀO!S Õ|2:Ö<ûì³J<!„|jµK–,aÖP4%„a‰ŒŒüYZZÚ<Š!„ï*33S2oÞ¼LŠ™Š¨Ù!#›9s¦çÏþó£r¹œ¢A!ä;ñxE„L%TóAÈÈÓÊd²¯)ñ „ò ¤R)V¬XqÀZŠ¡äƒr?I¿øÅ/hFAB!,!!áѨ¨(ê÷A(ù „ÜÏóIÉÉÉQ$!„<¨˜˜̘1c5E‚L5Ô烑¥~â‰'®x<žÇ)„BÆܹso÷öö*¸)dª šBFÖsÏ=÷ÜM !„‡õÓŸþô+ÏQ$ÈTBíÒ Y2…B1‹Â@&+ŸÏ›Í†O?ýðÌ3Ï@£Ñ@ !cìÇ?þñì«W¯*\¢hJ>!ƒ&O?ýôl ™lÜn7víÚ…ŠŠ øýá“*s‡ÄÄD¼óÎ;P(ƒ®ßØØ©T •JEÁ$d„üèG?š+Â}>m—LÔ슑O>è1™T\.–.] ³ÙŒ`0ˆèèhFFDGGªªª°dÉØl¶ëoÛ¶ +W®„ÛMÍÒ IJ¥³gÏÖP$ÈTB5„Œ ÈÈÈ¿úé§çR$Èd²sçN¸Ýn¨T*\¼xJ¥2ìóîîn¬[·v»éééhkk ûÜb±P QQQˆˆˆ˜C‘ S Õ|2²ÉÇÓ4¹ ™L‚Á Ο?8~üø€Äè{úzñâE€ÍfCss3Ž1 P(ÐÛÛûw 2•PÍ!#èÎ;OÊd2 ™4Ün·ØÇã^Ç®R©D\\œN'þö·¿è«¹zõ*BíÑ/]º„Ï>û /¼ðBX Q[[‹úúzx½^H$èõz¬]»Ç ø>—.]Â3Ï<N›Í†êêj¸\.H$ ¬];ô¤Ï^¯ÕÕÕhmm…ßï‡\.GBB‚Ø|ŒÉfÆŒô ˜BÈ!“T*eXVVÖwZÏl63^f³Y\¦¥¥…)•ÊA—S©T¬½½=l›V«•`F£‘åç纞^¯g===¾Ïùóç™D"t£ÑÈ6™TnÞ¼ÉfÍšu‹.­„B(ù SFQQQXBPXXÈšššî{³~ýúufµZ™L&cØÁƒ™Õjeׯ_gŒ1ÖÙÙÉA`˜Á``V«•9VWWÇôz=À¤R)s8’Ð6F#³X,Ìjµ²üü|ÆqÀ þ‹Åb?ËÎÎfmmmÌáp°ªª*1ùÙ´i6™t¾=7 !„J>ÈÔa2™Ô‚ÀâââXQQëêêr]…BÁ0«Õö~bb"Àâââ$2@@L@’““$X^^Þ€}…jC¢££Ã¶J0îNJBIR¨vçîïH%„B(ù d´µµ±œœ1™¸û•œœ,ÖjÜ/ù¸yó¦XÑÖÖ6èþšššÆqœØŒªò1ؾêêê¦P(¼'‘HXooï ûÊËËkR¡äƒñCÎ !„4 4 8§Ó‰+W® ®®‹UUUhnnF[[¤Ré=·e³Ù !•J¡Ñ >MV«… ðù|èèè€V«?“J¥ƒv€ŒŒð^kk+€¾ó•••ƒîËãñ„-K!„’B!„B¡€B¡@ZZ‚Á ÊÊÊ““—Ë…={öàðáÃ÷\ÿ³Ï>½çÜ”J¥ðù|fU¿ßzý…&7´Ûí0÷\–fŠ&„J>!„Œ“ÊÊJØívhµZ¬Y³fð ÇaË–-ðx<ÈËËC]]Ý}·Ëóü˜ÿ_t:222î¹ÌwIj!„PòA!dÕÔÔ ¼¼›6m2ùY¾|9 ¨¥ÌÓO?  ¯¹S00ŸGh;.— ‘HøÿÐbÏ´´4*TB™ÀhâB™Æôz= ªª v»ýžË^¾|@_“¬ûÑh4bŽÐzw»té‚Á A€Z­~àÿC()jnn›`Ýmß¾}X¶löìÙC…N!“0ùxÀ)ïèpƒŒŽB¯¯›Z°€„EBÈxظq# ü~?bccQ]]=`™`0ˆcÇŽ¡¨¨öy¨)Shæs ¯ÙU¨ TNN΀¤ÀívÃd2222­.NFƒ`0ˆÌÌÌ53v»………hmm½ç,î„BF_ÄwXV°@òŒ3~mÿÙÏ~†Å‹G-Z´è1¹\þÄc=E!½7¯×‹Ï?ÿüÆŸÿüçž?üá·jkkg|üñÇ?¸sçN-€C)J“Ú·##2y´¶¶â¥—^G„’ÉdˆŽŽ†T*…ÛíFkk«øYVVŽ=¶þºuëPSS©T µZììl$%%ÁëõbÅŠ°ÛíÉdÈÎΆ\.‡ÓéDII <T*ZZZĦ¾¾±±±P(p8¾ëPŸ·¶¶"66>Ÿ*• J¥°Ûí(--…×ë…^¯Çûï¿ÿP‰!c~£ñ]ï×™R"""\Ï=÷ܟΟ?ÿÉýf½%ßMOOÛ¿ÿÍÇÜ3sæL+5r“:ù dÒ¹ví3ŒçùAkmår9+--r¾‰D".›-~vãÆ –šš:`{DZŒŒ q~»'ì?Çp?okkcZ­vØû"„æù dêû|.pöÉ'Ÿ|úßÿýßgÆÄÄÈ)d£' âøñã“Éøæ›o~í÷û÷Rd&]òAQ “VhÎO>ù@ßHW?þñ‡œ«£ÿz¡¹=T*Õ€æMn·|ð¼^/$ ^xá…Aç ñûýp»Ýà8.¬#ùp?úšY}øá‡ðûý÷Ü!“âFj>F’@·yåß¾d æïÃáýöåÐüíëÒ·Xò¡š1cÆ“ÉäÛ»w祥zì¸\.lÚ´É×ÖÖÖùå—_þ €›¢BÉ!„J>Èw&°@êŒ3ž|þùç?ŠŸ¥R©fÿð‡?þþïÿþñÇüï(L÷öÅ_üŸ/¾øâëîîî¿Y­Öž .ð]]]?øæ›oþ€¾nõ›|è"##_VVÖ»qãÆ')äããŸÿùŸ¿þõ¯í½uëV ;E„’B!”|aÛ @­Vÿ×[o½5;11ñ9Ð4#Æï÷ã7¿ùcïÞ½soݺõÉ;w¶èxäC=kÖ¬+ï¿ÿþœ_|q…v|™Íæ;Û·o÷~õÕW:J@(ù „Bɹ/€SO<ñ„äÝwß±råÊRHFÏ·Ýþ¶cÇŽ·oßÞ`/îÑmàî¡v…Y³fU•””DRâ11ÆÈýû÷?&ÂÿµI$„B¹mDDD]AAïñxT”xŒ>Žã°mÛ¶G].×æÎûöþýû£(ñ˜˜xžÇñãÇçEEEýÔYŠB!$Dù¿ÊÊÊîPâ11üò—¿ä80göìÙµ¸«Û@(ùXõä“OþpóæÍ3)\תU« T*°–¢A!„DDD˜7oÞùóŸÿü ŠÆÄ‘••%¤§§?2kÖ¬S’žçÿ¹¸¸øq ÓÄ·k×®¨ùóççQ$!„B°`Á‚åǧ¹:& HçÏŸ¿}“;ö%‹èáꯟþù\A(JÜ·3önß¾ý$úfš$ õù „2b¨Ïǽ͚5«£¢¢bÑúõëgQ4&¦ººº;ñññÿ}ûöíggXõÜsÏõRâ19ð<_|1 †¢A!„i,aÁ‚2J<&6ƒÁ¹hÑ¢(|[û1#22òg›7o~„B3yÄÄÄÌ]°`õû „BÈ´Åóü¯Š‹‹©ƒù$°{÷n‰ ;€›={ö–,Y2¦Í].*++ÑÑÑ—Ë—Ë·Û ¯—ZÝD"ÁìÙ³ñÅ_üÀmÍjAM°!„2}wîÜYI¡˜øâãã#ƒÁàó¤cL¡P(F}§>Ÿ'OžDUUìv;¡×ë!—Ëñýï2™  %¯÷ãõzñùçŸãƒ>àÛÛÛcjkkc?þøããwîÜ©p@#E‰B!SÜšçŸþKžççS(&>žç¡Óéüuuu1Ü×_ý„L&ÕVVVÂd2A§Ó¡  111à8šªâAH$H$<óÌ3‘–¼ýöÛðù|ø×ýWÃÞ½{c{zzÚÀ«:(Z„B™¢7³ëñ‹_ÐSëIÄ`0Hš››W͸sçÎÌÑšXÐív#66ÅÅÅ8wîΞ=‹U«VQâ1ÂAÀ¯~õ+É_ÿú×':ôÂܹs?àyþ-Ðd„„B™‚"##•‹/¦ùé&µZ¹sçþdÆhíÀn·céÒ¥ˆ‹‹CSS´Z-E}”q‡mÛ¶Íüä“Oæ._¾<ç‘GùO2Š !„B¦’o¾ùæé±è6@FÎÓO?`0øô¨$X±bJJJ››K5cL.—£¾¾^xíµ×ž‹ŠŠú€Š¢B!„©b,º ‘%‚ÁàœŒð¤hX½z5Î;NG‘gf³ùÎöíÛ½_}õ•€"2êh’AB!#†&¤ëíTáõz±`Áÿˆ&>Ÿ+V¬€ÉdBZZEy‚(--ýfçÎÿíóù~’—~ !„PòA×[@ww7ÊÊÊ`³Ùàóù V«‘’’òÀÐ=¤RéˆÿgƒÁ |>ߤ6""#Úì*33Z­– &++kFZZš4**ê4EƒB!8yò$–,Y‚'N@*•B£Ñ µµ+W®ÄæÍ› ¿Óöš››±hÑ¢ÿž^¯K–,Íf›q±ÎõõõhmmE{{;ÍPqqñœßÿþ÷1‡#ÀŠ!„B¦«Ë—/###YYY8pàúüZYY‰M›6A=ztØÛüä“OFeÂl¯× »}ê´œ±š={ö   £5l/y8<Ïãøñãó¢¢¢~ ‚—B!ÓX^^4 ><àÞ5%%&“ ¥¥¥âMcc#.]º¶œÛíFYY|>l6ûæy.++Cww·ø¹ßïÇ‘#G°cÇ;v ~¿_ÜF¨Ù×Ýúo£ººpéÒ%qÓ>ù¸|ù2¼^/6nÜHGó¶jÕ*(•ÊG¬¥hB!d:r¹\hmmEjjê#²nݺP[[  ¯‰VQQQØ2v»F£~ø¡˜˜Íf|úé§âç«W¯FEExžGQQ–.]*Ö\½zF£qÀþF#®^½ ·Ûªª*€ÅbÁÕ«W)ù€¢¢"äççÓÑ< ìÚµ+jþüùy B!„LGÝÝÝú&½ŠB¡€D"AGGÇ°¶™––†¼¼¾Û+«ÕŠ5kÖ„m«¡¡o¿ý6šššàñx°wïÞamW£ÑàìÙ³€ 77—’ŸÏ‡ÖÖÖ° “‰+!!_~ùåó$ B!„LWQQQ÷ü\„ÙÏk¯½&þ[&“!99555Ó6î|\¾|ÑÑÑ#V@dtñ<_|1 †¢A!„é&4\ígŸ}6ä2~¿.—kD†Í½»†E&“M©äcž|ÔÔÔ 11‘ŽäI$&&fî‚ ¨ß!„B¦µZ ™L‹Å2ä2¡¾ƒA|ïî¡woݺ5¬ýù|¾°¿ƒÁà=“šÑ1k"yèQº»»Ç¼£¹ËåBee%:::àr¹àr¹àv»§|aT¶?{öl|ñÅ?p@3€ZÐ䃄B™8ŽCvv6òó󑜜Œ¤¤¤°ÏÝn7L&T*V­Z ¯åˆÓé [n¸ón\½z âßõõõâ$†¡ïN§ …†ÝÏdÚ&ýƒ5š|>Nž<‰ªª*Øív$&&B¯×C.—ãûßÿ>d2Ù¤Ÿõq,x½^|þùçøàƒøööö˜ÚÚÚØ?þøø;wjÐHQšÜZ[[qáÂ8N19÷x<”œ39—H$ …*• :111“æ÷…ÊŸÊŸÊú–?¾ÜÜ\Øl6¤¦¦";;ñññˆŒŒÄG}„¢¢"øý~\¼xQL¢££QZZŠC‡!!!|ðJJJÝö™3gð /ˆ›L&<ñÄP*•8tèš››aµZË—/Ð7eEnn.<¶mÛ6èÔµµµH$Ðh4“>þìað<Ïz{{Ùhª¨¨`r¹œ¥¦¦²ºº:9===lÿþý7üqÏÌ™3­Ôô³4>çÓƒêêêbÙÙÙL.—3•JÅòóóYyy9³Z­¬««‹Ý¼y“ôa¸yó&s8¬½½ÕÔÔ°ÂÂBÏA`qqqìüùóò÷‡ÊŸÊŸÊê–?F—ÖѹÞvðàA¦P(Bqf‚ °ŒŒ æp8,›••Å8Žc˜J¥bUUUL¡P°k×®1Æ»~ý:S«Õ +((`V«•`ùùùL"‘0L.—³ššš°m—––ŠŸK¥Rf6›™N§cUUUâ2‰‰‰ Óëõ“ú< Ͻ‘Ñrýúu¦×ëYtt4kjj¢_ÆQØoûÛ¯çÎ{‹çù·@“Nøäãúõë,;;›I¥RVPPÀºººè@½½½¬¢¢‚iµZ¦T*\8Æ •?•?•ÿØ—ÿùóç)ù˜‚×Û7n H8†:®_¿>¬m†’‡ÃÁ€˜¤ e8ûŸì&tòÑÙÙÉd2+**¢šŽ1víÚ5ÓóÈ#ü€Œ~ß&fòQ^^ΤR)3™Lôds ÕÕÕ1µZÍ’““Ç5îTþTþTþÓ£ü)ù˜X- ¾‹þÉ™àÉGCC“H$aÕMdì½ùæ›·£¢¢þ @E¿qçÇ0°üü|¦T*Y{{;¨ãô$4''‡©T*ÖÙÙ9¦û¦ò§ò§òŸ^åOÉ%”|ŒròÑÞÞÎd2khh šN:œ;w®‡‰ñcXrr2ÓëõìÆt€Ž³ÒÒR&—ËÇì”ÊŸÊ?99™ *ÿiTþ”|LÞ䣷·WlrEþçxžPmú}>6lØ€¢¢"q22¾ŒFc¤ßïlçÎÿÛçóý4$ï¸Úµk¼^/Þÿ}q2~¶nÝ X¿~=šššF}D**¯×‹÷Þ{Ê–?™|xž“a'›ˆo3Çß@Dfýþ6lØApüøq*™ &;;»×l6_¾uëÖ:ŠÆ}ŸÄŒÊ†ËÊÊPTTD¹ hÇŽp¹\8wîܨíƒÊŸÊŸÊz–DDDè~ŒÐõÖï÷Ããñ@.—‹ïƒA¸\.Èår¸\®A× }–¹?§Ó)Õìv»Ã¶=ØþÝnwØv§Ë…oç‰ÑìÊjµ2¥R9êÃö’¯:\¸pá—è÷î¾?†#®««‹I¥Ò1o_N†~¨ÕêQ‰ÊŸÊŸÊú–?¨ÙÕˆ_o­V+S(aï9€]»v) ¦P(˜D"a‰DüûÚµk 3 ÎQ|;¼®Ãá°íÁöÏó|Ø~âââ†=ŠÖdov5c¢A{öìAAAÁ “ªñÇó<Ž?>/**ê· !xÇÜÞ½{‘ •ŠºÞLÔóãàÁƒÈÉÉA0¤ò§ò§ò§ò'“”\.‡Ãá€Ãá@NNrrrÄ¿CµÝÝÝðx<â:•••P*•ßi?Z­VÜî7 T*‘žž>-búˆ’ûñù|hmmÅš5kèL˜ðå—_>€z<Ž¿ß‹Å‚”” Æ$9?áõz©ü©ü©ü©üÉ&“É`µZÅ×Ù³g‡½®N§›^UWW#99yľÏóhmmÅÚµ}L T*l6:::Äûåµk×âèÑ£HNNFVVÖ­[‡O?ýIII”|ÜÏåË—ýÐ#<ÏãÅ_ ˆ¡hŒ¾+W®@£Ñ@*•R0&Éù¡×ëqåÊ****2q…B!¾î5:Õ`Q[[‹ºººy€ît:áñx Ñh ¬ÿPèßw¿JR6mÚ„šš¬[7ñ%÷䣦¦æ¡ÚÉ‘±3wÁ‚Ôïc TUUÑù1ÉhµZ466RùSùSùSù“),)) EEEP«ÕšEƒA8NñÕXÝ¿ß/~ÞØ؈ 6Àd2çy ”••èkÖÕÝÝèèhèõzœ9sÎö IDATpæ̼ùæ›xå•Wpᤤ¤ 777¬#ü„MüÆû twwSGóIF£Ñ 22ò9Šdðóc¤Ú}SùSùSùOßò'£G"‘@«Õ†½ª¹êo° CËèt:(•J¤¥¥…-Ëóü€‘«4 8¶žçÅexž‡Ñh'®<}ú4¶mÛ³Ù Žãðî»ïBœ:u ™™™(--ÿöz½ÈÌÌĉ' qêÔ© ÿqŸdpáÂ…°Z­4ä$ÒÑÑø‡¸~ãÆ¿£h 0¢“ Òù19Ï 6 ½½ÊŸÊŸÊŸÊÿáoÔh’Á1¹Þ’1J<""Æ¿Ù•Ûí†L&£Ò˜DA@0œC‘ óƒ ~~ø|>****BÈ Æ=ùðûý4„à$#‘HÐÛÛK…FçâüÉÑŽ¨ü©üÉô,B¦ªqovõ°ë“q:p¨x(#Z LçÇä=?F¢Ü¨ü©ü©ü§oùÓõvl¯·dìÎB!„2¹\.Ô××OŠQ¢¦ ŽB@!„B¦“ÐLâ~¿ …Ðjµ8}ú48ŽnGÕ|B!„i# býúõÈÎÎF{{;.^¼ˆ®®.øý~9r„4Ê(µ#d +++ƒÓéô3‰D‚×^{mL¾‡Çã“Yš/\¸¯×+Ž»N!„ÜíÒ¥K!ìZÁq\Ø\ÇŽÃÁƒ!$ NŸ>M£ÏêpN¸Ü@à3¡:œÇÆÆÂf³A£Ñ øL&“áìÙ³£ 6@¥Ra÷îÝ£¾¯ôôt8NX­Öq??¨Ãñôþ}¤ò§ò§ëíĽÞîÝ»ÝÝÝâ„|¡™Æ¾sÇaݺuhoo‡ Ø¿?l6NŸ>MQãù¡j>ü~?E‘ N£ÑŒëÍxss3T*!„ ã8ƒAñï+W®Àl6ÃëõB"‘Àh4B¯×C@RRJJJ(p#ÿ‡Y¹²²’"HÈ$çv»qéÒ%¬Y³û÷ï‡L&ÃöíÛÁó<º»»QYY §Ó •J…7†U;—••aÍš5øàƒPWW‰D‚¤¤$±¦¥¬¬ >Ÿ6› ÕÕÕHJJx½^œ9sËåHJJ KPª««ñÌ3ÏÎœ9ƒ`0ƒÁ€µk׆}w›Í&~ž’’B…I!ä¾bbbPRR"Î¥“––†´´4Ô××cÏž=–œƒAê„>‚ªÃ¹Ùl¦2ÉÙívF¼ôÒK(//Gqq±xÓ¿xñbX,ƒAœ8q‹/Fss³¸®ÑhÄ+¯¼‚œœø|>”——cÙ²ehll#BÉGUU€¾êí%K–ààÁƒðûýhllÄ’%KP]]-n×d2aÛ¶mX½z5\.š››}ûö‰Ë;v Ë–-CGG<Ö­['î—BŠV«…N§Ã+¯¼‚ŽŽ@kk+Ž9Aƒúúz¸\.ÀÉ“'¡×ë)p#ˆ=(³ÙÌfýoë12ù †È#|>Æù¡×ë™L&cF£qÀëÆŒ1ƬV+À²³³cŒõöö²ëׯ3žçYVV–¸­@ Àôz=S*•aßO§Ó±@ Àc¬§§‡I¥Ò°õ +((ÿ6 L­V³žžñ½‚‚&‘HØÍ›7Åu$ »~ýº¸L\\S©Tâ~A`&“Iü¼³³“ñ<Ïôzý„8?&Òv•?™|åO×ÛѽÞV\\Ì4 S(L­V³ÂÂBñÚTSSôZ-Ójµ,###ìšEîx~¨šQ†‰Ïï÷‹éú¿úW)À–-[<ÏãòåËðûýxã7ÄÏ9ŽƒÉdBww7l6›ø~jjªX-ÔjõýÁ¼^/êêê••%¶¥€­[·ÂëõâòåËâ{qqqaM¼´Z­¸ÝË—/Ãçó‰ßT*âãã©À !„ÜÇqxýõ×ÑÖÖ‡Ãööv¼ñÆâµiíÚµhjjBSSŽ?vÍ"{ !SÛp;œK$ñßÝÝݹ\¶Ì³Ï>+&!ßå9´^vv6²³³|Úo( ºßvîîÈ®V«Q__O…N!„PòA™,B ÅÝì<ψl¿´´kÖ¬¹g4Üd¦ÿ:>Ÿ B™Àh†sBÈÏ?ÿ<„5ƒú&fâ8nÐyC†C.—C*•Âf³A¡Pˆ/¿ßmÛ¶…Õ|<È÷£ç„B%„IF¯×C§ÓÁh4âòåËp:8sæ Š‹‹‘••õj(8ŽCcc#jkkÅ~#'NœÀþýûát:ÑÚÚŠÍ›7Ãn·C­Vk›jµñññÈÎο߮]»ÐÚÚJ…G!dؼ^oXSbBÉ!ä!Èd²°NÛƒáy …bÀæ/^D\\âãã±páBäää ;;—Q(ú|Èd2H¥Rñ477#99~¿¹¹¹(**BQQ.\ˆeË–A"‘àý÷ßûy„jHú“H$a}PΞ=‹¸¸8ÄÅÅaáÂ…hnnF^^Þ}ÿ¿„BHÈ¡C‡pèÐ! ÄŠÀCLOôM“>žë“q:p""BÇ ÇFòxž(ç‡ËåÐù|$¸ÝnH$’{v.¿¿ß¿ßÿû‹Œöù1åF¿“÷÷‘ÊŸÊŸ®·“ãzšTp÷îÝÑ1:?¨Ã9!ä¾F#ñ0"µ<Ï?Tò2•]¸pN§¯½ö€¾á/\¸0è² … *‰#„±´wï^TTT@x<œ={W¯^ñwtçÎÐjµàyyyyJ¥°Ûí(,, þÜ#Õ|,7Г˜ÁLÉš2>åö°ÛIOOG}}= ¬¬ F£Q¼p†øý~´¶¶B*•¢¥¥eÔ’M*:ÿ©üéz;Ñ®·¡š×^{ 6lÀÅ‹ÁqöîÝ ·ÛíÛ·cýúõèìì„ßïÇÂ… ÑÕÕ…ÌÌL¼õÖ[P*•°ÙlØ°a:;;©T†qŒ#GŽÀétÂb±@«ÕB¥RA*•¢µµŸ}ö AÀÑ£GQ]]“'O¢¹¹yÈÉuÉ@ÔáœBt:äryØìõ„2•••Áår…½g·Û±zõjð<”””°‰pF#Îœ9ƒòòrFƒA¬\¹N§ƒï¼ó•’BHÿÔ`0·ÛMOfÈœN'Ün7”J%cŠˆˆˆðš3gÖ¯_·Û=¢ûÚ³g.\8n7’ß6M"dX, *++á÷ûÑÜÜ ©TŠ?ü …[·n…F£ÁùóçÅåSRR`±X`·Û±jÕ*x<tww#77«V­²/ܸ6»¢!BF—Ëå›o¾‰sçÎaÆ ؽ{7ôz=fš;tèPXÇr—Ë…óçÏC"‘à7Þ M!qqqÈÍÍܾ}(,,Ć `µZ)@dZÚ½{7ÒÓÓQRR­V‹´´4p‡ªª*¬X±R©‰‰‰p:Ašš*’"“ÉŸŸ•+WB*•"99jµÁ`pÀ°õd‚%•••T„Œ¢7ß|¯¿þ:‚„innÏóp»Ý°ÛíÉdÈËËCZZÚ€ùUÈä&“ÉÂ8¬Y³Ç!''n·{Àˆs^¯~¿ÿž#Ñy<ðîÉh4R 2J®\¹FsÏeü~?ŒF#Þ}÷]444 ??Û¶mƒR©Djj*^}õUìرqqqP«ÕÈÌÌDFFššš`µZ‘••ŸÏ·Û ­V‹@ €èèè!÷ÕÜÜŒ÷Þ{---J¥8yò$®\¹­V‹††´··Ããñ ¹¹>Ÿ6› ½½½HKKCyy9ZZZÐÖÖ¥R ·Û]»v!::---hiiAqq1º»»áñx •JÁCBB ÷°jÕ*ŸO|Ïn·cݺuÈÈȲyß¾}ˆŽŽ[ÿÔ©SxꩧpæÌlß¾]|ÿܹsb­L^^ár¹ —ËQRR‚ììl$%%èkêb³ÙPZZJ…M%„q=¹9N|Zx¯eäryX[ÖпÝn·Ø¤ÂétB©T‚ã8ìÞ½[L vïÞýÀm®}>8ŽÃþýûQUU…¼¼>•••P(ðûýX¶l™øÃÝ_uuuØÓЋŗ˿ßòòrÄÅÅÁn·#11 ðûý°Ùl¸sçNØzN§‹-‚L&Ã믿­V ·ÛøøxTTT@.—ƒçy¬\¹rÀÿèkV2ÒC‹NR©4l¶rA P(M,”J%ŠŠŠàv»iØÈ)$Tæ¡Wtt4NŸ> ApòäI}5 ‹-ºuë`6›áóù$ù9óz½HMM…R©DzzzØÃÇ#žËý_ñññb-åpogÏžöþ÷¾÷=*hòÀ:„C‡û÷عsç=û(ú|¾ÉûdE¶R0DGG¼^/är9‘O&Œ˜˜dee‰Cÿi4”––†=!<{ö,Þ}÷]ìܹ»ví‚D"ÁÙ³gqéÒ%(•J¤¥¥èkŠU[[‹Ã‡cÇŽX¶l8ŽƒÙl†L&ƒF£ »1)))ÁóÏ??à ŽT*Eff&<’““‘””„gŸ}™™™°X,P©TÈÎΆÏç «QQ(ÈËËCll,8ŽCtt4¶lÙ‚`0ˆ]»vaÅŠ€ÂÂBh4¨Tª°õ+**0{öìi72ÎÝ_%%%‰ÍUóÚk¯…5½!SÏóà8N|@ðꫯB"‘ ½½]<ív;L&SXâêÿâv»±wï^qD=¹\Ž %%+W®Ä¡C‡ÄÏT*Ôj5N:¶ÖÖÖû6íŸ À§Ÿ~VûñÑGQ¡’IïîsãnõõõFÝš¬ØÃïõC“Ëå kooð¹Ãá`¼8Žc:ŽY­VqY«ÕÊ„½7˜ÒÒR&•Jö§R©˜ÅbaSÝ·ÿ_2ÂçÓHŸ999ìüùóc~|dgg³›7o†½gµZ™^¯—㵨¨ˆuvvŽéù1‘¶CÆþ÷q"l3aïõöö²¼¼<€™ÍfÆc*•ŠÅÇLJ-—••¶Laa!“H$ìÚµkaçÏóìƬ  €)Š°ßžçÅó.///ìïþ×Ú’’ÆcF£1lý—q8Œ1Æ ‹ŽŽf===Œ1Æzzz˜Z­žPçÊh\èÒ:z×Û‚‚VPP çjµšiµZ¦T*YKK ;xð ;x𠸼ÉdbUUU¬¦¦†©Õj¦×ë™L&c'Nœ`Œ1–ŸŸÏ4 Óh4,99™vóæMÏ4 S«Õ¬¼¼œ555±ÔÔT&—Ë™Á`·ÛÔÔÄ™N§c†¥¦¦²@ À™D"a999SâþqJ$555Œçy¦V«YvvöÉG^^³Z­Ìjµ2‹ÅÂJKK™J¥b‚ °®®®a'555 ËÎÎfíííÌáp°††f0Çqƒ&@”|LïÃòòrVXX(gcq~„~ðÆÚ`ÿÇñL>¾KÌGëæc<ÊŸLœ›Ïñ(ÿÁ¸…^F£‘1‰À’““Y^^ÓétL¯×3AÄ›®žž¦Óé˜T*e›6mb‰‰‰Œã8VZZ*ÞÀõOzzz˜B¡`Z­–q}AXjj*KMMe<Ï3½^/~á$]]]L&“1¥RÉŒF#S©TL¡PPò1ùmðåx%7oÞdqqqâñXXXȲ³³Ygg'S©Tbò.“ÉXOOKMMÏ綶6¦R©Xoo/“H$â6 XWW3™L,//1ÆØõë×Ytt4³Z­LvýúuÖÛÛËŒF#3›ÍÌjµŠû`Œ±ÔÔTVRRÂǸ]CGúxž2Í®Ìf3´Z- Š‹‹ñÎ;ï„uZ Q©T:«ÅÄÄ`ñâŨ¬¬öì¾çÏŸ‡Z­ÆáÇÚ†œ;wO=õÊÊÊ4yúÚ¬Þ«Ýl0„Ïç»ç2~¿Á`pÈNI¡aFÉÄÒÝÝ‚‚äççC£Ñ 99)))£ÚTO"‘àÔ©Sc>ëê`ÿ§»;§Ž¥‰Ðr<ÊŸLïó°¹´xžÇóÏ?¶ßÜÜ\¨ÕjÔÕÕ! Âd2!!!µµµˆŠŠÐ×wÄjµ¢²²ÍÍÍP(ؽ{·8œ÷Ý£¤ ‚€‹/âOú“8 ÕjEuuµØ™Ýl6#))IümÚ²e âãã\³Íf³xMS*•hooGYYº»»‘ŸŸåË—‹£Ü‘IK  @!€ÿÀ¿¨Ð=V_@"‘àðáÃ8räœN', ´Z-T*¤R)Z[[ñÙgŸ‰£6=zÕÕÕ8yò$š››á÷ûÁó<´Z-/^ ƒÁ žã­­­â=¡L&CKK êëë¡V«mÜdÈÄÄDX,¬]»–ª­&RÍÇ7Çqbf@|swÍG¨ y°ïª1NÍGVV“J¥ìúõë>kii³ák×®1…BÁ***ħ3‰„ ø~ñññŒã8±ùVMMÍ€íêt:ñÉUtt4kiiaŒõ5;ËÊÊb‚ 0L–‘‘Áz{{©æc‚œOƒ>Ôh4C>¥'ßSçÉ7•?•?•ÿô-ºÞÞ×î!jéÚîQ#òÀ15›ÍbÂPÍGgg'S(¬´´”555±’’±Ùâ‰'XNNKLLduuu,0µZÍòóóY]]kii «µkoogÅÅÅL©T²òòr¦×ëYSSSØý\]]]XMFÿšM›6‰ïŸ8q‚eddPÍÇDSVVŽã’’©T ƒÁ€'N {ªû²²2¸ï„lýmß¾åååX¼x1’““a0°|ùrq4‘þ5N§YYY8qâÖ®]‹ÊÊJddd@"‘`ëÖ­ðûýˆ…L&CSS$ Îœ9ƒÄÄDX­Vèt:¸ÝnÄÆÆB¯×£½½ÇaçÎx饗àp8päÈTUUáâÅ‹P(øðÑšš …B1ìÚœü± wj4ŠÁ†ˆˆ€ÍfƒÍf›ÒOÄ'ÊP†ãyŒGùg®—ñVè«ÍjµµÃ)ÿ3fL‹óŸ®t½ýïk¾}°¨‰‹ÅǃíÛ·‹`~øá‡P(â}Ù›o¾)Ž˜’’"Nf¹jÕ*¸Ýntww#77‚ àÍ7ß×–.]Šööv¼þúë°Ûíðx<0 8yò$´Z­8Î`5“!uuup¹\Éd¨¨¨@VV–x-ë– Tó1µZÍRSSÃÚÖkú×|èt:f4ÅW¨†F£k †Ûá¼³³“mÚ´I¬m=Å*//°ßPg¦þ®R©³ZŽãÔ¢ ±½~AAAlªUILLdííí,##Clob±XÂb0 Obè5Š/“É4©Ÿ|^»vMlãÚ¿Cßtyò9Êÿ~ëwvv²âââq‰Qè)ŸÙlÐ)únyyya©üÉT/zÝ÷õþÃœL«Õ2…BÁRSSYOOëííeÉÉÉL«Õ²øøxVRRÂL&SXíeÿV5………L£Ñ0ƒÁÀJKKY||< ¬´´”EGG3­VË222Ämggg³èèh¦ÓéXCCkkk ë<^TTÄ, ³Z­L©TŠƒ+„ZÊôöö2ƒÁÀâââ&ýùñmá=pæñ\¿¹¹+V¬@AAbbb·oßF||<6mÚ$]æt:±páÂCpJ$èõzlÙ²El_W__ØØXX­ÖaµQƒhmmÅ•+WPQQ›Í†¢¢"äææŠûmjj‚V«×9yò$222póæMìܹUUUÈÉÉùÚívܸqëÖ­ƒ×ëECCàßáòåˈ‹‹ƒ ˆ‹‹ƒÁ`@BB¨=MŒˆˆÀ·í3I¸‚‚‚‚!k>î7ùÝÌ™3‘””„ÜÜ\h4š‡>?ÆS}}=öìÙ«Õ:킈ˆÜ} §ügÍš…Ÿÿüçxíµ×F¤üï·~YYêëëï;ÄãhHOO‡^¯‡s¾—… ÂjµNØZœ‘*ÿ™3gâå—_F^^Þ¤?ÿ§³ÁÊÿ!/*Óáz«¿GíGÈçÞPà €ÀTz{{YUUÕ mt3 L*•†í÷îeÍf3À®]»ÆŒF#“Éda52ý_Œ1¦×ëïÛÖ/”E«T*ñÿwßjƒ:~5‰CµùÀ{ì1–ŸŸ? öëaΖ–¦Ñh˜^¯gjµš544ˆ5m¡¡õz=»~ý:»ví3 Ì`00µZÍ ËÏÏgZ­–Éd2ÖÒÒöTH£Ñ0­VËn޼ɪªªØ¦M›˜N§c*•J-D£ÑˆÃöª04Â\tt4‹«Í›Žmþ°'žxbÄË¿ÿúd999aOúëbU&“±¢¢"vãÆ Ç¢££™Z­;L&KLL‡D‹‹cƒi4ñ©œN§c2™ŒÕÕÕ‰OÕjµ8ÔdOO Ìh4ŠÇžF£af³™UUU‰O‹‹‹™Z­f:Ž) ÖÔÔ$çªÕjÙõë×ÅãG«Õ2ƒÁÀnܸ1éË4ÎB}>&™¡ú||à8€U87Ý”<~ÆsdÈ1¬œ¼ÉGoo/aÐ1›ššñ†g$“žžÆqœØ¤änùùùŒçù°ýÞÝy¼¨¨ˆ ‚À뛋!”¬ô×ÕÕ%/¸iÓ&¦V«,c2™XCCs8ⲡ&/ƒAÜ%3ùP©T¬´´tÈæüÈÈÈo Cç“ËåbR^RR"vdãy^lÚ¢R©Äs'tãjµZÃŽùÄÄDVQQÁÌf3S*•âÿ!::š544„ý€†š]…öJ8L&Ó€sc:%£Yþý×/((‡< L¡P°ÎÎΰ&OYYYbÓ¸žž¦T*™Ãá`F£‘eddˆ¿Iè7—Rtt´¸N¨SdCCÓh4âP“&“‰²òòr±ihûý›]õôô°¸¸8ñØ›Œ1¦P(˜Ãá`]]]L©TŠËŸ¥¥¥HMMÐ7ëðÁƒqòäIlÙ²E\fõêÕP©Txï½÷òòrÔ×׋MÁ.]º„ââb$%%aÛ¶mb³,Žã —ËÖÖVJ& ƒÁ€œœœQx:@[ IDAT:/99©©©8þ<âââ°qãF\ºt ÇaÇŽú:¯Ùív}Ãÿ…:×Éd2<÷Üsúš&z½^èõzp‡½{÷Ân·£±±‰‰‰N'vV«Õƒƒ~§«W¯B¯×‹MŽšÊôöꤨP(àv»Tõ+•J¤§§‡•W¨|ˆ¿-¡!UA›»>óÌ3hllÄ•+W ‘™™ p¹\à8.—K<^A€Á`ð»zøða;v N§uuuaMUûŸW_}@_çN§ÓIå?¹ÝnØívH¥Ò°¡x ¹‹À‰~Mª‚’©‹‹ˆˆ¸ #'cÏy³Ù ¥R6ºT0™L¨¯¯ñ¶ÂG…ËåBjj*²³³¡V«áõza³Ù V«qôèÑ°åC}S4 , xžÇ[o½%^Øsrr‘‘óçÏC¡PÀb±Àï÷‹óˆ¤¤¤ ªª /½ô’““Áq***““­V‹·Þz ±±±X¶lôz=œN'jjjPRRBGùÁó<ŒF£Øž´­Y³‡Cì‹T^^“ɵZÝ»ÿgàŽã†Lú;sæ <ˆ¼¼¼°±ù¿ÓÎ]ûr»Ýðù|ÓbdŸ±.ÿÁb?ÙÙÙxöÙgÅ¿¥R)êêê¾óvt:rssÃþï{÷î…ßïr§Ó‰•+W"??iiiP«Õâ|ý÷­R©Äã7 :Ÿ•ŸôôtÔ××Ãáp {ŸÏ‡½{÷âí·ß~èý‡ú"ƒADGG£¥¥….d(GÆkÇðù|P«Õ⃱`0·Û-> êýþº»»áv»¡Ñh†œmº›Áóü —Ë5)¿¼ÑhÄéÓ§‡ü<-- f³‚ @*•Âl6ã…^¸ïvC“©Tª!—‘H$hhh€ÅbAFF t:***ÐÖÖ6`Ò˜òòrFøý~˜L&´´´„-sàÀ444@¥RÁï÷#''a7eçÎCUU$ A@UU8ˆŽŽF[[RSSáõz¡V«ÑÒÒ2ìá†ÉèËÍÍÅ©S§ÆìÆcÆ (++CBBÞxã ¸ÝnÄÄÄ µµ>Ÿ …eeeعsç°¶×ÜÜŒ¸¸8$%%AX,–aÝÈø|>ñÔ××Ãår! âÕW_Å|@å?NA€×ë…ßïG||<Ο?¹\žç±bÅ tw÷Ñ,×®] ‹ÅŽã P(°gÏ:tñññ0›Íðù|p»Ý¨©© [/4¬äÖ­[¡V«|îñx°|ùr´¶¶Âï÷‹Çï( i:mÊ0û÷ïGQQшl«´´ …7nܘ–O‰Ín·cÉ’%HOOÇž={°hÑ"ìÛ·Oü=Z¹rå€u†z?”t¬X±¯¼ò öíÛ‡E‹áÈ‘#èÁ†EDD¸\.—l²Œ"rwrq/R©4l™áŒªô59î²kÖ¬Áš5kî»\dd$¶nÝzÏd@§Ó‰Íîuaªª^©T†=m$ÓÛ;＃ÌÌLTTT€ã8œ={2™ f³éééâÍa(íßÄE£ÑˆsrÈd2¨T*$%%!==ýÿgïþÚ:Óüñ¿Ñ Q¢“Ö´M§éšÎÄ1Î@¶0…K˜Ò‘®tÁŠ+^âFlq¥ßê§ºÒ SœÒ-Ž2¥«ŽºÂŽŽ0…?•+^àG°â‚BGiÁš–8D$šçû‡“óÅ–‰¼_×u.•„“x?'çä>ÏóÜ Ñhœœ ™L•JÕ#Q×ëõP*•ÐétP(Xºt©4 Q­Vcûöí˜?>€›«žw¶HƒË5DóÖó»«}õz=²²²°fͼùæ›Xµj‚ƒƒ™™™0 =*ºVïíë8qíÏ`0 ##sçΕnŒ¬]» …  “ªò¹ö«×ë___Ìš5 *• hhh$%%aáÂ…8|ø0¶oß. ÓjµRï0ÝA¤•ÇÊn·ßqM«Õ*=nµZ¡Ñhú|¾Åb\.¿íÚ?®„´¿Þ6‹Å•JuO¬y@ÃË5´=++KºY­V<ûì³Ðh4ýÞ¨Öjµ}ö$ ‚€¹sç"%%Eúþh±Xàç燙3gºýM‡açííýÇmÛ¶uzjµ+wçšp~§5C<µZõâÖɳ&œ²ýÙþß…Édê±êrJJŠ¸bÅ 199Y”Ëå"Q£ÑHקM›6‰J¥R jµZ±°°PÅ›k"„‡‡÷(”PZZ*í·°°P SSS¥}jµZQ.—‹r¹\ª\æt:Å””Q¥RIûÒjµbqqq¯‚,®çÈår© ˦M›¤Çe2™çVUóÀ çn½ÍÍÍÃÃÃ{ý¼¸¸X ›››E•J%†‡‡‹AAARµ½sçΉ!!!}*r­ÝÖ]mmí=YÑñ»ÏcœNgÉîÝ»/ð …&“é®î. 6«ÕŠ-[¶Àl6£ªª ÇŽƒJ¥ÂÂ… QQQRQ€;wbæÌ™°Z­ ÔÖÖâÌ™3ˆŽŽFdd¤TÐÄn·£¦¦………ؾ};ÒÒÒ°sçNèõzèõzéïëׯǖ-[°}ûv477£¾¾Z­&“Iš ¶a䦦"##ÍÍÍØ·oöíÛ' ݺu+RRR––†ææf>|555ROÑ@´´´ô9_ø‰'ž@]]ôyÙ¸q#ªªªƒ×_‚  ¯© gÏží³ ç}ôM ¤ªªj›Ãáð˜‰{žD¥RÈ^DDD½.ú‚éºÞ§¤¤ ..---ÐjµÒ=×½õë×Ãn·ãý÷ß—†H½õÖ[¨¨¨@vv¶4ïÒáp`Û¶m=†»žïÚ—Z­Fff&¢¢¢¤ç¬X±ÑÑÑ0›ÍÐjµØ´i¥!ÊZ­™™™Òü£ŒŒ $%%aùòåÒã;wîDpp0*++ï8t™È¥¯Ý‹XhµZ)¡ˆŠŠÂ–-[°víZ7+EfggKǹJ¥ê1¿‘îœ|ØÆŒÓpèСÀÌ] """Ϥ×ë{Üh|ä‘Gnûüšš( é‹V÷/n·V#»Ó¸ö—^z v»%%%hiiAcccjj‹f³¹W‰eW z³Ù,m}8~ü8“W•Ñ[œ>}:­´´ô¾ˆˆˆ± ¹¬Zµ F£qÀàìY³¤Òœ=ö˜4lêêꜜ,­“C}«®®Æܹs¥»Ñ‡MMMÈÌÌ”î$»«… bÏž=·M8V­Z…Í›7£¨¨Þ:Dt:Ìfs¯ø666ÞÕPmA˜˜ˆÈÈHüán äååIÏqU®r%<Ý墢"¼úê«€ÈÈÈ‹ ‚€šš.bHwåÃ?Äúõë ‡Ã½^ ‹ÑÑÑxï½÷°jÕ*øúúbíÚµ°Z­ýîoÛ¶mHOOGpp0A‰æœßÛ‹š:uªÕét²Ú±úÆ·wÏU»2™Lbnnî°¾¦V«›››¥jqݵ¶¶Š …BL&“´iµZ1%%…ë<  çC­VKq‹µZ­¸eËé²k}…ÖÖVQ£Ñˆ¢Ñh³²²DQÅ3gΈFt:Òïvttˆùùùbxx¸Ô†111â®]»ÄäädqÓ¦M¢(Šâ±cÇÄœœñܹs¢F£[[[¥»Ô&“Ilnnår¹Ôƒ•””$ffföøWUU‰ÉÉÉÒÿÉõ:Ý{>rssE“É$ž;wNT©TÒët¯ã@¬¯¯EQÓÒÒÄ+VŒŠžÃ‡Kku¸þÝ}}WÛçææöèIÈÏÏÓÒÒzô$>|X\½zµ˜””$nß¾½ÇóÏœ9Ógïhiii×s:âöíÛŤ¤$155U<|ø°ÔV®öqõ’äääˆIIIbFF†Ô+Òýñ-[¶ˆIIIâêÕ«¥ýÜkŸ^oÝ«çÃétJçúnÇó­³6-N§3fÞ¼yÿ§¶¶vœ^¯çü"µ`Áìß¿éééhhh@KK‹4¡3((Hsíëë AðÊ+¯ ¨¨kÖ¬Acc#G±×Ï<ó a6›±wï^ÄÅÅIíß¿›6mByy9d2***¤Úÿ®^Š£GÂÃÃïzÂx||<®_¿Ž´´4DGGãwÞaän’Zøøx\¸piiiHKK“æîTTT@­V£¸¸ÀÍ9!®úö®R¨:r¹\º+={ölÈårTTTÀf³I½c‹ˆE\\öíÛ‡ÈÈH,Z´„ X³f €›ó5\¯£V«¡Ñh¤¿ß:á8((‚ `ýúõhjjBEEEIŸÝ=zF£Q_½hÑ"$&&JûvÝÑ×jµ¨¨¨íkÕž¾ªø¸Ž‘[Ï}í«¿JR::®×Ïo­ )“É°dÉ’^sŽn}}¥R)•Òí¯ÅUŠ—h¸Èd2Î߬XöÕ³äp8–>þøã;>ýôSÆï1LDžgýúõ¨¬¬DRRâãã±xñâÛ>ñâÅÉdˆ‹‹Ã+¯¼ÿ^Ï1™LÈËËCqq±4!¸¡¡IIIøä“O¤ê72™ Ë—/—ªÏ¬]»jµûöíƒÏ]ÿ_\Οzê)øûû# €¥ @©TJ±{â‰'ˆéÓ§KÃSÖ®]+M^·n]ŸV‡ÃÑg²ÙãK£B¡€J¥Â©S§päÈäççK Î †_d2Ù€†Í!-- ©©©=Ö†èï‹A÷äEé}s +"ºõ¼Ö½ J¥º«Jp®uqèÛÓß9ßétF‡‡‡wíܹó†‰ÈóTWW#&&QQQ=z!úSWW‡—^z ³gÏƉ'z•inÞ¡ÌÍÍ…\.‡N§ƒÅbÁܹsñ‡?üz½^z^xx8  Õj!“ÉÖg•¢¢¢>_çv_¦·lÙ‚äää»ú=ºyw:-- p8ˆˆˆ@QQ´Z-¥E²\st<¥R)õNtO<öíÛ'=¶téRìÞ½‹/F^^æÌ™ƒµk×Âb±`æÌ™¨««ƒÝn‡V«ÅîÝ»ñòË/ßñýZ­VÔÕÕ!<< ,€R©ìÑcqkÏÜÌ™3Q]]††@vv6ÇfQ¿×Ç3fHóg̘3fH‹YöeëÖ­ÒÚ6=öƒøÜnìCEggçÓK–,)Þ¸qãçÅÅÅþà?˜Ìy†µk×båʕصkôz=V¬XÎÎÎ^wk är9²²²°råJ( !99v»z½^º#®Õj¡Ñh¤!W'Nœ€F£Á믿.í/""ï¼óV®\‰ÀÀ@À¦M› Õj¡×ë¥U ''S¦LéuÇ=((Hº[ݽÌ.ps(GRR z”Û¤Þ‰Ú­‹¾-_¾555Ø¿?¶mÛ†•+W"88°k×.©çª¡¡³fÍ‚B¡Àûï¿ß«MæÌ™ƒÆÆF<ûì³R-_¾V« ÈÏÏäççC­V#77W¢¥Ñh¤ÉÃÝ'w?.“’’0kÖ,|òÉ'HHH@XX4 ’’’àp8 Ñh„ùóç#%%z½*• }ô‘4ù]§Óõù:jµºG¢LD£SPPTÒ¸Yåõ×_ï·Ìw~~>¦OŸÎÀ ¯&(¯zyy­zá…š222¦êõz雂··7:::¾õ¢_^^^¸9ÿ„<êÀñòèñ3Úˆƒy<»ÛçÃápÀÏÏÇŽë‘D|[kÖ¬ÁòåË{ÝY¿>ƒÑn#ÑþaaaX·n]¯¤FGû“û·y½üëmEEÒÓÓ{$EEEزe  ƒ4´7!!r¹ùùùÐjµØ¹s'üýý‡††X­V¼ÿþû ÂŽ;••¥R ¹\Ž={ö@,]º‚ ÀjµÂápà“O>µÃ¶¼¼¼úvÕàmQ EEEU3fÌ&Ož|&..®bïÞ½u>ø`ç­+ѽ§¤¤>ú(%ñ€—^zéžK<ˆˆÈýY,äåå!//[·nEjj*¢££íÛ·¸9ü³²²ï¾û. 6nÜ(õ(ÇÆÆ¢¶¶©©©Ø±cššš––†òòrTUU!66/¿ü2AÊE×ÖÖÂh4ö([?ÝMw…ÀJ+¯^½²{÷î»wïö?~¼`6›Çrâ ѽmÎœ98uê”44g0ð¼á~ºß $"ºWÙl6i™R©DVVæÌ™àfqWeǘ˜˜>G÷¸ž;}útäççãÈ‘#=*î-X°©©©n7uU„S«Õ£~*Ù·ü½Ê¿oèêêúcccã‚1<”‰îmƒ™x½^;wöùXbb"òòòPZZŠmÛ¶ ì õ-UüºWÜ£ž¾sÂàt:KvïÞ}¡$¢ÁàªVDDD4\•AèQ ¢½½½ß^‹ÐÐPTTT ¥¥ÀÍŠ{œ;7DÉ€’ªª*Å­‹C7+ˆÜC‡úñ°DD4túªØ«²£Éd’~f2™°bÅ ÔÕÕõH*\ûÒh4ؾ};.\ˆàà`˜ÍflÛ¶ r¹¼We¿Ñ>äxPª'L˜0á³?ü0ðÖÕLôXÍÃ3VßèÏ=]íêÛxì±ÇÐÜÜ<àç ‚€ÀÀ@TUUyìâp¬vÄó#ÛŸíÏë­g]oo½͘1cÐ*;RÏãyPæitttlX½zµ•!%rv»Ï=÷üýýŒ¹s確ºv»óçχ¿¿?üüüðöÛo¸ÙE¼xñbâ±ÇÃÊ•+ܬ†àà`øùù¡¨¨°páB,^¼^^^¨¬¬Ä²eˈàà`̘1---X¹r%, .\AðòË/ÃÏÏþþþX¸p!ŠŠŠ°xñbLš4 Ë–-ƒL&CHHˆô:DDDÃ¥¤¤?ü0L&!2X“Ä NŸ>m---ídH‰ÜCvv6|}}Q[[‹?üÕÕÕp8Ø°a´Z-jkkQ[[‹ÂÂBTWWÃf³Áápàرc¨¯¯ÇöíÛa³ÙðòË/#66UUU¨ªªBJJ ¬V+, Ôj5œN'”J%AÀ±cÇPUU…ˆˆ`ãÆP«Õسg ÐØØ(½®\.Gvv6ìv;*++ÑÚÚŠ7BBBz¬fMDD4æÌ™ƒúúz¬]»–Á"ƒ6 ¿££cuRRÒŽ¦¦¦)œÝO4ò***¤“§Z­–ƨVTT@.—#!!ÀÍ’#GŽÂÃà …*• 6› 555€êêj7<~ü¸ô|™L___,_¾o¿ý6šššPZZŠÄÄÄ^ï§{ÉÂØØXlÚ´ ±±± B¡žûÐC¡©©‰HDDÃÎU.—Ü<ù°×b±¬\»v­ßÛo¿}?CK4²är9®_¿.ýÛUB&“!..3gÎpsl«J¥Bvvv¿s,–/_.M[»v­tb?~¼”X,[¶ ©©©˜={vŸ ÊåòUB‡ôz¼aADDÃÍjµ¢¡¡¾¾¾.%o³Ù “É P(àp8PSS½^ÏRôwaP×æ¸~ýzìÆüqCK4²bbb““#-¤TZZ 0ÈÏχF£R©Äܹso[Þ6<<%%%ÐjµÉd†ÕÚsŠ×ñãÇ€E‹A«Õ¢°°PzÌn·Ãn·#""¹¹¹Ò𮜜DDDôùš§OŸ†¯¯/‘ˆˆ†„kžbzz:‘0 Åÿ²³³¥9‰Ï?ÿ<ÒÓÓqäÈ‘»®ì8š öíF‹Ó錙7oÞÿ©­­§×ëÇ2ÄD##>>V«sçÎ…^¯—¾Ì¿öÚkX³f ‚ƒƒIII Á_|Ñ£«9((r¹7nÄÊ•+ؾ};´Z- ƒ4/>>eee˜5kÔjuò„ÑÑÑF}}=ñì³ÏB&“!22K–,ÁÁƒ{ÔQ€ÊÊJÄÅű‰ˆhÐÕÔÔ ¢¢gÎœ‘œ5kJJJuÛß]·nô÷²²28NÈd2¬X±ï¼óƒ;CUºmž··÷ŽO?ýF£ñ{·},%è™KÿõÇmJíîرjµsæÌ ˜6m>Üç(wb³Ù†ÚÚZþ|°Ôêè>?²ýÙþ¼Þºïõ¶®®Ï=÷ÊËË¥›_f³ … xóÍ7áëë‹5kÖÞzë-444`ýúõ ‚F£Aee%6mڣѵZââbDGGcÏž=l™;ÏCy0ÇŽûÁ¶mÛ:¾Ï“+“ž ‡÷"ÖÔÔ„¹sçB¥RÁb±Àd2áµ×^sû¦§§#((ßfÝ ~ù$¶?1ùàõv ²³³‘‘‘…B£ÑˆøøxF¼þúëP*•xõÕWáïï»ÝŽ3gÎ`ýúõP(°ÙlÐjµˆïÑÖw»¦Õhþ|ŒÂýWtvv>½dÉ’ŽÇüó/¿üò C~o°Ûí7nÜ F½ét:Ô××cÏž=¨¯¯÷ˆÄ^zé%N<ˆˆÈý½òÊ+hmmÅûï¿N‡… ¢  QQQ(++CSSt:T*ZZZPZZŠ9sæ0pƒ`Ìï¿AE}}}ýt:3&&æhcc£¥û\cíÈsX,(Š«Œ„gP«ÕUMÊÓKÚíöeƒitaû³ýÙþîoëÖ­ÈË˃L&C@@Ö®]‹ŒŒ ”––" MMM())Axx8"""PTT‡ÃNÇày@ò€·EQ4U͘1C˜}:V¯^ƒB˜ÍfJ…YÂÃÑ••…ÐÐP„††"33³ßêŒÝOWI{ùäCúÎ `%€G¯^½úËÝ»w7DGGÛ.\¸ 0ùð,_ý5d2Ù9Fbèi4ðóáyÉù`Mêgû³ýÙþ£·ýièFäää 55Ó¦MóÏ>‹¼òÊ+€E‹! z½!!!0 X°`@«ÕJ ¦kñ^WÂ⪠I·7Rc1*ÿ¾¡««ë BBBÆ°9ÐÑѱaõêÕV6‡{¿ùÍo®õÕWëá¡P(‚¢¢"ÃìÞ½ÑÑÑl¶?ÛŸíOnÌl6£¥¥¥×6Ò‰¾ÍfƒÍfc — &œúä“O‘ÜÖ–-[ÄGy„ínoÐã^^^.êt:Ñétò tc—.]U*•xæ̶?ÛŸíÏö4¸Ìý _oCBBD­V+ªÕjQ.—‹Z­VÔjµbaaáˆGiiibZZÚmŸ“˜˜èÑŸ¢—DQS§NÝÑÔÔ4Å“D-ìv;¦NzãâÅ‹O¨cDn{2ô>ÿü󈈈ÀòåËa7µjÕ*8¼ûî»l¶?ÛŸí?h¼¼¼À‹QüëmEEÒÓÓ{T­rw^^^ŠïÃùþÝÊøñã+~õ«_]à=÷³hÑ¢Žx €çºáïùEQ¬­­ÕjµØÚÚÊƒÑ Õ×׋*•jÈÚ‡íÏögûÞögÏÇÐ]oËËËE£Ñ(ýûرcbDD„ôï²²2199YÜ´i“'‰z½^ÌÌÌEQ[[[ňˆ1 @Ôëõb~~¾(Š¢˜››+êõz1((H ÛÚÚD§Ó)&%%‰¾¾¾¢^¯SSSEQÅ€€Ñ`0ˆFÌÊÊ7mÚ$ž;wN ÃÃÃÅÑh4Š­­­âŠ+D=Þ³'ö|¸Û¤ööö¶”””\ãéÌ}lÞ¼¹ó{ßû^Ïu#“|ˆ¢(fddˆAAAbGGJ7ÒÖÖ&êt:q×®]Cú:l¶?Ût¶?“áK>DQu:xêÔ)QE1..N,..ÓÒÒÄÈÈHQEñêÕ«¢N§ëëëÅÄÄD1++Kú¹F£Ï;'êt:±¹¹YJDÊÊÊÄ]»vIûp:¢ÑhÏ;'>,vttHîš››E™L&Ö×׋¢(ŠYYYbRR’tù‹Å Œ€ÆÆFÃ`0`Ïž=®"l¶?Ûô¶? FN‡ÌÌLÄÇÇK?ÏÏχÃá€ÅbAYYBCCa4±wï^hµZÈårÌš5 ‹>ú(ìv;–,Y‚ØØXX,„‡‡£°°‡6› sçÎ…ÕÚÿ* ¨®®¼÷Þ{=Ê8³ïÐòõòòjòóókøâ‹/.stéÐ;yò¤C§Ó“Édÿ @ÃCð® [[eddˆjµZÌÉÉaÎaâ“«R©ÄÜÜÜ}/l¶?Ût´?8çcÈ®·µµµâŠ+zý|×®]¢Á`þ––&Fiù¾}û¤y‰‰‰b@@€ Íý)--ƒ‚‚Ä1""B*F™™)=×u uŸs’››+æææŠÍÍÍ¢Z­cccÅ   199Yšï•œœìÑå·=åx–xÍËËË:oÞ¼êS§N±ÜǨ©©¹4gÎœ¿Œ3æ<€%<§¹òá:qFFFŠ:NÌÍͯ^½Êƒyˆ&•æääˆjµZ4™LCRËŸíÏögû³ý™|¸Çõ655UÌÉÉé‘|ÜiýÁÔÜÜ,jµÚ{òó7[çãŽ=a^3iÒ¤Žù—ùzþüùÊéÓ§? Õj§Èï¥ÁpCÌn· ÍÍÍ盚š.½ÿþû×öîÝûHGG‡CÅ=6`Þw;û‹VTT`Æ ¨¬¬DPPbbb0}úth4h4 ¸ ®nõ––œ8qÅÅŨ««CDD^{í5 ·{Ïl¶?ÛÿÞm®ó1¼×[‚€cÇŽIó,ÒÓÓ`Øæ÷´´´ ,, ÍÍÍ÷\£yyyyìÁ`ß¿'%L>î"ÿ`þûV`7¸p G'ÝKìß¿ÅÅÅhiiÙl†Ùlî1qnO.—C­VC«ÕB§Ó!&&¡¡¡1ÙíÏögûß[ío·Ûqß}÷9A˜ÀÖžë­Õj…R©dÒÎäƒÈ3Œ3æz{{û8vÄõM<÷Üs8pàƒqŸÜþÍf{„ў䃆>ùÃ0 ™LÖÆ 4DDý;tèÊÊÊÐÔÄ îwb±X “ÉþÆHн„ÉÑ êêêú²¥¥… "êG~~>   €Á¸ƒ³gÏ¢««‹Y1ù ¢¾ ‚ðõ7ß|ÃÕ¿ˆˆú>G¢¸¸PXXÈ€ÜÅbA{{{ #AL>ˆ¨ßkÅW_}egˆˆz;tè´°Z]]‡^ÝÙl¾~ýúu3#AL>ˆ¨ß䣹¹ùÃ@DÔ›kÈ• ‡^ÝÞW_}åÀ‰„Ä䃈úO>Ξ=ëdˆˆzê>äÊåOúsf³¹ ÀyF‚˜|Q>?yòäd†ˆ¨§îC®\Nž<É¡W·IÖN:5@£AL>ˆ¨?5—.]âÅ”ˆè·¹ráЫþ“µÉ“' ÀÊh ¾ŠŠ 7Wµïþ÷êêj‡É‘gq8{ßÿý«ŒÑM‚ àÃ?¼«¤d´+**r^¾|™™ÙIOOØl¶ÏÎÎfp†ˆÝnǸqãnp…s¢Á÷Œ^¯ßuêÔ)5CAD~æÌèt:ª›x Ãjµ>€]é}ðòònܸ1V&“1¢©© cÏÑà;ÒÔÔ¤¼ul3Ñhu§Þ ½ê©¦¦7nܸÂÄ£r¹¼ÍlfbOb6›!—ËÏ3]$|//¯?øàƒ^|ñÅñ vƒiiinŽµ¯¨¨€Ñh„ÑhhµZ©›âââ.™L¶›‘èߘ1c¾6›Íj;žã믿†L&33ù N§³äøCè‹/¾ø}FƒˆF»åË—÷ø·+ùX·nƒÓ‡={öt\¼xñCF¢]]]_466ú‡„„p‡hll„Ýnÿœ F44ö?~|"«^Ñݨ¬¬ÄÅ‹¯¨d4ú×ÑÑñÁÿüÏÿ\d$<*©¾vùòå"&DCÃît:·fÍž‰ˆhÀ^{í5Gggç0wTR]]=Éáp0 ¥¥çÏŸïPÍäƒhˆ‚ðÖ¾}û¼êêê ""º£½{÷â¯ýëß®^½ºÑ¸#»L&û¬¤¤„‘ðEEE˜0aÂ'×ù J‡Ã±&))éCADD·#V®\é°Z­¿d4¦££cýŠ+®‚À`¸1›Í†õë×;ÚÚÚÞdòA4ôvœ8qÂ~ðàAF‚ˆˆú•——»Ý~/·ÿÒ¥K§víÚÕÉP¸¯M›6uy{{ ÉÑÐÇËK—.eïõÉápàW¿úÕ¶¶6£qwÚÛÛ“ÿýßÿ½ƒkk¹§¦¦&lØ°áÆßþö·ÿÏõ3&DCoïùóç¿Ø²eKCADD·JOOǸqãà$Á»WsãÆÿŠŒŒlçð+÷b·ÛqÃÇÇçWÌL>ˆ†Ñµkׯ\¹òZuu5ƒADD’½{÷bóæÍW,K"£ñí´··¯mllü,))‰¥¯Ü„ ˜?¾³½½ýƒóçÏ¿Ûý1&DãÑápÄÌ™3§Ýl63DD„ºº:$$$8._¾ Àˆ|ûﺗ/_þ—÷ßßüâ‹/:Ù2²ìv;æÍ›×YWWWg±Xßú8“¢á³ÿÊ•+é?ÿùϯÚívFƒˆh³X,ˆŒŒ¼áíí·”ï¼W®\ñÿðëFãuÎ---øéO*œ{Æ u^^^SÒÓÓù9$" 'A@RRR×ÇÜÒÖÖň +W®½råÊ •Jõüš5k~û‹_üâ?ûÙÏÆDGGcúôéÐh4Ðh4Éx (‡Ã‹Å‚––œ8qûöíßÿüç.NwzâĉËOŸ>]>ý0âD#ÃríÚ5CvvöÏ?ÿ\—››+W(Œ Ñ=Êjµâ…^¾øâ‹š¶¶¶gpòßðÄý#«ÕúE[[Û’­[·Æ_¿~]{áŸóçÏã°¬?~¼¨R©œ>ø ÝÇÇçË &l¿víZÞÉ“'ïª[‰ÉÑ& —/_ö?pàÀ®   ÈÒÒRFèÝcyÀöo¾ùf9#2"ìGÍÍP|;ׯ_Ç×_¯¿þú;í‡s>ˆF–påÊ•X³ÙüƒÁÀu@ˆˆî1%%%xú駯_»víßΞ=ËăF=&DnàòåË¿¹pá Ï<ó̵­[·2 DDN¤§§#..îjgggh[[ÛF…ˆÉ‘;ÙíÚµ'ÿã?þ£~ÆŒdDˆˆˆÈ3’Žûî»™™™8sæ ¢¢¢"" V«±mÛ6TVVÂÏÏOªšEÄ䃈ܒ~ùË_â›o¾Á’%K8‰œˆÈ ?~¿ûÝï0a„˜|‘{™0a&OžŒÝ»wãí·ß†\.gPˆˆ<œÉdÂÉ“'1mÚ4øøø0 Ä䃈FžB¡ÀäÉ“qèÐ!±""ºÇèt:üå/Á‚ 0qâD„˜|ÑÈqU²ª««ƒÁ``@ˆˆîA2™ ;vìÀªU«˜€“"“&MÂ?ÿó?£ªª jµš!"ºÇ¥¥¥aãÆœBL>ˆhxùøø`îܹصk'•"/¾ø"6mÚÄbòAD×xøúúbÇŽ Ñ(M@RRR˜€“"z“'Oƾ}ûØãAD4Š¥§§cÁ‚¬nHL>ˆhèLš4 Ÿ~ú)T*ƒAD4Êýþ÷¿Ç#<Â@“"|'NÄž={àëëË`d2JKK9˜|Ñà'¯¾ú*æÌ™Ã`‘D§Óáw¿û!$&D4x Þxã ‚ˆˆzILLÄ~ô#‚˜|Ñw7aÂdggs‚9õë¿ÿû¿9üŠ˜|Ñw÷è£"66– "¢~ ÄÆÆbüøñ 1ù ¢og„ غu+ADDwô›ßü†A &DôíÂh42DDtGjµÑÑѦKL>ˆèîM˜0ï½÷ADD–ššŠqãÆ1Ä䃈îÎôéÓ¹¦Ý___øùù1Ä䃈nìرˆg ˆˆè®-]º“&Mb ˆÉ ðƒ;f bbb""ºkQQQp: 1ù ¢Q«ÕÐh4 Ý5•J…àà`‚˜|ÑÀüâ¿`ˆˆè[‹ŠŠ‚AL>ˆèöär9‡\ÑwŠ1cø5˜|ÑøøøÀ`00DDô­ ‚À@“"º½'žx‚A "¢ïìÿñv\â’ÈÃhµZ<–ÝnG]]¾øâ ÀøC ( ‡h˜ýä'?Á‘#GbòADý›:u*ƒ@Çb±`Íš5ÈÏχÃáèy!’ÉwÞy§ß亲²*• z½žÁ$$?þñ¡P(`·Û 6vEÄäƒhH™Íføûû#77‚ &“ &“ €ÂÂBøùù¡®®®×ï/[¶ ³fÍ‚Åba0‰‘N§ÃøñãVìù ò cÇŽeòAgÕªU°X,Ðëõø裠Óéz<ÞÔÔ„çŸHHH@mmmÇKKKD¢!àãã///‚†{>ˆ<,ùàâ‚äIA@qq1`Û¶m½àæÝ×>úPWW‡êêjŽhhµZttt04¬ØóAäA:;;¡V«ò‹Ešãq»cW§Ó!""---¸|ù2€›="GŽ‘Æ£ïß¿gÏžÅÌ™3{$1‚  ¤¤°ÙlP*•0˜3gd2Y¯÷³ÿ~üð‡?DHHêêêPTT³Ù ¥R‰ððpÌ™3§ß÷i³ÙPTT„šš8h4DEEIÃLj< ×ú "¢ÛE‘7ÚT* ))é®~/77·ÏÏ@nn®ôœcÇŽõÙ›z½õõõ=öY^^0™LHMMíó÷ŒF#®^½ÚëýC©Töù;&“ N§“í=€ À:"€uŒÇÈn—.]¸qãxa%""&Üî-33³GB‘‘ªªª;~YommEyy¹Ôc²iÓ&”——£µµ¢(âÔ©SR‰Þððp”——£¹¹eee0•J…æææ^ɇkŸ&“ ¥¥¥(//GjjªÔS’‘‘Ñã½”––J%''£¶¶ÍÍÍ(,,”’Ÿ¸¸8¶7“Ol"""&Üî­-%%¥×±¬P(ÌÌLœ9s¦ßßu•ß-//ïñóèèh@DDD¯DÆétJ HLLL¯äV¯^Ýëµ\½!=öåJ0nMJ\I’«wçÖ÷ÈÉ“""bòÁÛlµµµX±bE¿kyÄÄÄH½wJ>.]º$õDÔÖÖöùzUUUn®#âFÕ=ùèëµÊÊÊÜœˆ{ëÏ”J%:::ú|­Õ«WK=)lk&L>ˆúÇ çDD4,  6n܈––:teee(--…ÕjEaa!ª««Q[[+õ$ô§®®‚ @¥RÁ`0ôùœ   iµ††I©Tª>'À;¶×ÏjjjܪUPPÐçkY­ÖÏ%""&DDä&´Z-´Z-âãã!òòò°bÅ ˜Íf¤§§ãÝwß½íïŸ={¤9ýQ©T°Ûí½VU¿ÓïuçZÜ°±±&“é¶ÏåJÑDDL>ˆˆh„ ±±AAA˜={vß"™ K–,ÕjÅêÕ«¥aN·#—ˇýÿ‚ÄÄÄÛ>çn’""&DDDƒhß¾}صkâââúM>\žzê)èÕKÑ—©S§¸9ÜI„^ëy¸öc6› ß¹Ñ}aÏøøx6*ÑwÀ•eˆˆhȸ*N¢±±ñ¶Ï=xð ô;!½;ƒÁ ÍçpýÞ­öïßA P(àëëû­ÿ®¤¨ººZ‚u«·ß~HOOg£1ù "¢‘°hÑ"hµZ8„……¡¨¨¨×sAÀÖ­[¥õ@’““{<îÊäZù¸9ìÊ5jÅŠ½’‹Å"•÷MLLì³gd BBB`0 –.]Ú«g¦±±¨©©¹í*îDDä®ј1øéOŠŒŒ áäÉ“¸xñ"Kâ pµÒ¦¦&|úé§X½z5üüüú¬ÞBž‹Ç97OÛŽ;Ö£z•Z­Fdd$L&"""z<Ö×*è‘‘‘nN7(,,”Îwz½^ÚgFFrss‘––&íS¯×÷X­ÜUj·{)ݾV@¿õñcÇŽII^¯GVVrss±zõjiH—Ñhä*ç,µËR»DžÂËË O<ñŠ‹‹yòäíêÕ«Ø°aî¿ÿ~x{{ó`còÁÛ°oç΃Édêw¢¸F£Á–-[ú]¯£ûœääd鱶¶6ÄÆÆöÚŸL&CbbbÄã»$®uJº—ë½ÓkqcòÁ䃨ïüîð&|ðAüéOBhh([d ‚€mÛ¶!%%]]]šÔIî™|y*ך§OŸ–¾¸ÿä'?éw­Žî¿çZÛC¯×÷Þd±XpôèQØl6(•JÌœ9³ÏµB, d2Y‰ä}¸9Ìêĉp8·}-ê÷fã:iÒDQä$7¸ùK4j’1cÆ %%ëׯÿNãqéî˜ÍfÄÅÅ¡¶¶W®\a@˜|1ù`òAƒð½öÉ'ŸDdd$ôz=~ô£áþápÿý÷38ÝŒØ7þ±cÇ"//‹-b+ 3FƒŠŠ ¼ñÆøío‹ööv…ˆˆˆè[ðõõÅ›o¾‰èèhÃ]“qãÆáÀxúé§Ù#è׿þ5~ðƒ`ùòå¸víBDDD4@S¦LÁ‡~ˆY³f1waØKíŽ7999L<Ü„Éd† ¸*/Ñxyy!-- V«•‰‡»'ãÆà /¼€¥K—2òn$)) ñññðñña0ˆˆˆˆú1vìX`ݺu †'$ßûÞ÷°sçNFÝ eeeáÁd ˆˆˆˆú “ÉPQQùóç3ž|Lœ86lè·Æ;,¹\ŽmÛ¶±÷ƒˆˆˆè®BI!!! Æw4lõÕ´Z-š››q7g0pâÄ ±Ô.õ{Q÷ò Éds§L™òxWWףׯ_W߸qã{‡ƒwþî@.—;Æm̘1ícÇŽm¹zõêñëׯW8$Š¢í>fØøˆQ||F—ïÿû8{ö,פó¤äcܸq¸pá+*y׊½×¯_g0˜|‘{iT{{{¿!“É~ñꫯŽ—ët:f®‹EEEí©©©¶¯¾úêzWW׿‹¢¸—ÉÇè0nÜ8äççcîܹ †'%øì³ÏmñóŸÿ` ˜|‘û~a\4nܸÿZºtiWFF†’=ã´´ôâÂ… í6›í3Kï…áXL>noêÔ©hiia ÑO8;v,/^ÌH{ÐÐP<üðà ‘û}Q”yyy½9eÊ”ß?~|rNNaqkkë?ÄÄÄüØËË«^n IDATÚËËKϨܻär9²²²ˆÁ> õ Lœ8%%%0ÃöŸ2›Í(((@CCÌf3Ìf3, l6[ü”J%Æ‹/Âét2 nˆ=D£7ñ7nÜû~~~3KKKP©T Êzë­·þºvíZQÃEQlôàãŠÙooo\¹r…•ZÙÏœEZ­vÈÿ#v»;vì@aa! £ÑFƒGyjµšîÀf³áÂ… 8zô(êëëQRR‚¿üå/èììdpˆˆFøû®¯¯ï?UWW?À‰¯#oÍš5?ð×µk×{yyÝ˱F«'Ÿ|’‰ÇP$¼CýcÇŽ…ÝnÒÆ+((@JJ BBB˜˜ˆÐÐPV$äÄî÷¿ÿ=Ö¯_«W¯²Gd„±çƒh^¬½¼â|ðÁ¬Ó§O«x#ͽ̛7¯áƒ>hEq¾‡[lÄ>Èårüö·¿Å²eË OK>†òË’ÅbÁÂ… a·Ûñî»ï"((ˆ-:„AÀ¶mÛ’’‚®®.8…É ý—Cݸq㎟8qb²^Ï)îÆápàá‡þÊf³ý›(Š%L>î >>>Ø·oß°N-Æxêoll„¿¿?"""PUUÅÄcÈd2,[¶ §OŸÆSO=…É“'3(DDClìر¯¯X±B`âážär9öìÙ£3fÌï¼¼¼8ìâÑÕÕ5,ÓF#ìù¨¬¬ÄóÏ?íÛ·sáÂôÆoà·¿ý-ÚÛÛŒaÄž¢Qt‘öò2øøø²Z­“9öܽiµZóÙ³g“=m ö|ô›ôù´ÑÊãz>0þ||ôÑGL>ž­ç&¶lÙ‚U«VÁn·3ôy""7¿@{yɽ½½­ß|óËêº?‡ÃI“&9AxØ“*_±çch¯·MMMÈËËC]]ìv;|}}±`Á„„„|«ýY­V Åù@Øíöa© ëQ=K—.EPP7“””„øøxøøø0DDƒ'ôG?úÑ&žA.—#((è*€PFƒ`ÇŽðóóÃöíÛ¡R©`0PSSƒY³fañâÅá®öW]]iÓ¦ úû´ÙlðóóC]]Ý°ÄÅc&FUTT ¦¦õõõ<šÝPVV>þøc4773DDƒ#&>>žwu<ȳÏ>û½êêê§ìe4F·ƒ"11IIIظqc!\ˆ‹‹ƒB¡ÀæÍ›¼ÏÓ§OÉ‚Ù6› ÷N¦Çô|¤§§#--ãïÜ”\.ǶmÛØûAD4H&NœèȲ‚äÉ'Ÿ7yòäF‚V¯^ ƒÁ€wß}·×w×  %%[¶l‘¾ôWVVbÿþý=žg±X——»ÝŽºº:TVVòòòÐÔÔ$=îp8ðÞ{ïaåʕغuk¥\þnÕ}EEE€ýû÷K¯1꓃Âf³aÑ¢E<šÝØ3Ï<NÇ@ Qµ,õéY¦NŠ®®.6Ú(g6›QSSƒØØØ~½~饗%%7—†Ù±c233{<§±±&“ V«'NœƒÜÜ\|ñÅÒãÏ>û,òóó!—Ë‘™™ ©‡äÈ‘#0™L½^ßd2áÈ‘#°X,(,,”––âÈ‘#L> 33©©©<š=Àš5kðÀ0DDßÑ7¦¨Õjƒ( tvv²ä(×ÔÔðõõí÷9Z­J¥ Úg||·‹Û'D@@À 5 -¹\Ž§Ÿ~š "¢Q™|8fŒ<gÏží÷9‡f³yPÊæÞÚâV«‡uù=—|ìÛ·ÑÑÑ<’=Hhh(~øa‚ˆˆˆF___¨Õj”––öû×\ððpég·–ÞmooÐëݺΚ ·Mj†¢bÖÝpûR»MMMÃ>ÑÜl6£   0›Í0›Í°X,#ÞXž’í?/^d0ˆˆˆhÔ‘ÉdHNNFjj*bbb0oÞ¼[,¤¤¤@¯×ã™gžpsäHKKKç tÝ#GŽ **JúwEE…´ˆ¡kÂ{KK \,:ÏdÔ&݃5”ìv;vìØÂÂB466"::F£<òÔjõ°¬úèél6.\¸€£G¢¾¾%%%øË_þ‚ÎÎNçQSSƒ½{÷¢¥¥EJέV+“ó&çJ¥ …Z­z½!!! õ˜ó Ût·? L_«†³ýG—×^{ uuuˆErr2"##1vìX|þùçÈÌÌ„ÃáÀG}$%زe ²³³…£G"''§Ï}ïÞ½3gΔþ’’‚)S¦@§Ó!;;ÕÕÕ(//<õÔSn.YñÚk¯ÁjµbÙ²e}.]QRR¥R ƒÁàù Šâ·Þär9:::¾Ó>î´åççC£Ñ 66eeep:Cúz£m»zõ*6lØ€ûï¿ÞÞÞ<#àçéÛngÎœArr24 ôz=RSS±k×.”——ãÌ™3¸téõl—.]Bss3êëë±oß>ddd 22 …(..vËóÛdÚÿæÇ<ÍßÛÍc>ÿ4t×[§Ó‰M›6õ¸‰®P(˜˜ˆæææ^ÏMJJ’’½^ÂÂBhµZœ;w¢(¢µµUšß‘––&%©©©R«Ñh°oß¾ûÞ²e‹ô¸J¥Bnn.BBBPXX(=Ç5ÅÁh4ÞÇÜw}ƒCõŸomm…ÑhD@@ªªªxqâÍétâ¿þë¿0qâD.é!ÉGkk+’““¡R©––†3gÎðX‚­££ùùù ‚N§ëuá©í?²íÏäãÞJ>ÚþÅÅÅL>îÁëm[[[¯„£¿ã¡µµu@ût%ÍÍÍp:R’Òß6׿gŽ9wL>N:µZÌÌLöt óvîÜ9„††bòd.ÜëÎÉÇ®]» R©’’Â;Ûø•••Á××111#w¶ÿÈ·?“Ñ‘|ŒôçŸÜk¤ÁÝlÝ“O:ÏÊäãðáÃP*•=º›¸ ÿöúë¯ß±6 ÿÉÐét"55:õõõêëë¡V«qøða^äÜ`Û¹s'&Näb°îr2t:ˆ‰‰ÑhD[[ÑÞ¶lÙF3l Ûßí.Ðü&? “áþü“ç&Ò+O:·ÉFÓAd·Û±páBdffJ%Èhd™L&8¬ZµªWj~kÖ¬ÍfäIo4r^zé%ÀܹsQUU5äqØþDÿ{ïÓÖ•ç¿ioZÏ3îÊU]­g×ÒÜì8ªÑš£1[1£:Žâ(ä(fC„Q@! Q@…,dqž Uè†(tK¢š' € ÙÂÆÙÂcG™>0‹+˜Wa…ç‹wñâgñ ·c·çùƒõY!)¿ ç%])¹>÷žËùqÏýœóù¼ÏÎíÿŒÄC$mˆ"ìZ“´Q–ãk?`RÒšYæ¹¹¹‹Å¸uëk±[Œ’’ØíöUo¨³ÓgbÖƒ––ÔÕÕ±An RVV¿ß{÷î­[¬þ·à”DØÌtBÖ!I‰Òÿ—“fü°ñVƒAÈårz.Âï÷C.—Ãï÷/{]ì·˜,ób|>•jq÷^.ÿ@ wß7¡´Ü®œN' źËö²ãõ—wïÞÍÞtëÜŸÈ RªR©tÃã رúþ¡R©ÖM‹Õ?s»bl=·«êÿŒµoN'xž_¢4ÓÓÓày<ÏS##öÿééi »ž?ÿŽäu§¦¦–Ü{¹üc«"±|L&ÓªU´Öûxc§4 Ë—/Ãf³1‰×-ŠH$­[·Xú&qåÊ”””@©T²ÂØ¢ý£¡¡¥¥¥ˆF£¬þ Öÿ Š\.ÇÔÔ¦¦¦PZZŠÒÒRúÿØjÆää$‚Á ½¦££ …â•òÑjµô¾³³³P((((Øe°#Œþþ~„B!?~œµú-̇~øÊ‹ñÃñxÖo¼u:qîUSSSXrO›Í›Í¶ìw/Ïó˜Euu5º»»iÚÕÆ| †%çyž‡Ûí^òƒÆ‰<MOO$ Ìf3š››×¤ým{·«þþ~h4šl126‘H„¿ýÛ¿e±A<~üjµR©”F‚ôƒÁ€Ç³úg¬ “É§Ó §Ó ·ÛÙÙYdee¡²²‚ °ÚÁýŸ±¾pG¾yž_Qj9²²²ÐÛÛ‹¾¾>dffþàçñù|ƒtÕsñJLìßÏŸÁÁƒ155…¼¼<ôôôàСC?¼l¶{å÷ôôü ?9ÆƳoß>Œcff†Æ:ÓÙÙÉúG‚¡Õj188¸&®¬þwæQVVº»»è‚  ££‚ @¥RáøñãqrŸÑh]]]„H$ÂáÇ_¸gV4EGGD"²³³Wuýàà €…€Ü¶¶6øý~h4äççÇÉ„ܾ}~¿F£‘õƶ$;;EEEÐëõKÜb£Ñ(|>_œa*“ÉâÒ‚@Óøý~TTT ¼¼"‘F£---Ô­krrƒmmm8~ü8ÚÚÚÐÞÞ±X ³ÙŒüü|üô§?]“ õmo|LNN²@óC­VãÍ7ßdÁúãý£©©‰Õ?㵈F£èëëƒB¡ †G0Dzz:A€Á`@4Euu5êêê011±XŒp8Œôôtø|>˜Ífúÿ†††%qÑh¹¹¹p8xøð!ý:pà<Ìf3¢Ñ(öïß’’\½zpûömx½^LNNRŸòææfôôôàþýû€‘‘ddd@¡P@­V£²²rGí_°–ýŸ±~H$’%q±•«Å,·A`,^¯‡B¡@~~~\Z‘H´D¹J­VãÚµkqù‹D"šF$ÁjµÒ+ïܹƒÓ§OÃn·ƒã8Ü¿b±Ÿþ9ŠŠŠÐÔÔDÿ …PTT„ææfD£Q|þùç‰Q ›óÁó<¦¦¦˜ÿpcccxÿý÷ÙÛkúëÛ£¨T*Vÿ,æã•c> ‘ÉdD*•’‰‰ š®¶¶–H$òìÙ3zn``€ ‡ƒ¦‰Dq×UVV‘HDæææâb>, ‘ÉddtttI‹¯w8 „bµZ ât:ã®@fgg !„h4¢×ëI$!„277GT*ÕŽˆùXëþÏb>6v¼eÇŠùK–¢[±XÌ´ÌYÿ`¬Ð?Âá0«ƪÛËb¿sFƒp8Œ³gÏÒvTUU…§OŸR«p8Œßýîw´ .zf³9n/˜ .`ll,.¦²¬¬ øòË/ãµÚÛÛa2™â®ÏÌÌÏóèêê¢çäryÜìðÞ½{é3ƒAŒŒŒ °°®vˆÅb²þÏ`$Û~­R&!™`H$ÌÏϳ‚`ýƒñ‚þ …Xý3V…^¯_â&188ˆôôt|òÉ'¨©©¡ñðz½K‚cC¡T*Õ’¶¸8&Äçó¡½½ðèÑ£¸ô“““§¿?oúQ²‚ Õää$à§?ýiÜù´9æZöc³xƒc+¾\™ ƒÁ>>ëgð‚\.GAAu¡Õh4‰s“Éd8{ö,úûûWõÜ …R©tÉsôõõ±JelYü~?\.Wœ¡¾ÓáX0 ƒ±=ñxýƒÕ?S»z<Ï/9”J%ÉËË#ccc4ÝÄÄÑh4@Ôj5éëë#ƒØl6š®§§‡( €pGrrrÈÌÌ !„††¢×ëãò///'J¥’LOOÓë•J%ÍG©TR5­Xúçïáv» Ïóô1,±XL…BAª««7mwõV»Z~ËXûñ6‰@©TÂn·ÇËÊÊBCCÃŽ·%mTå½.III›z=csHJJb…°ýi­ûGKKKÜFG‹‘H$Kôÿ׋`0¸!»t?xð¡Pˆê®ofÿX‹vÀÞ[þ=H6²~B¡8Ž‹S¯zQ‹Å¯-V‹YX¬þ:lUµ¶ÿîWI[½ÿ³ñvýÆÛÞÞ^Øl6 Ç—<ÏãæÍ›hhh€X,†D"Á;wvŒú 3>Ìø`ÆÇk_Ÿ‘‘Ç'©C&“áîÝ»ë^¹¹¹P*•¨©©Y÷¼ àóùàt:™ñÁØvƃŒµo¯\¹‚ÉÉIª4çóù¨á!‘HÀq:D¥ª?ùäx<ܹsgG”ë–v:cŠG ÆÖG­VoêÇøÐÐÐŽ’Úd0 Æÿ¸æ¸¸ýÊ?~ »ÝŽP(‰D«Õ ƒÁ@W³³³ÑØظsÊg+?\GGkÁ F‚ðèÑ#dffâ“O>L&Ù3g ‰099‰ŽŽø|>(•J?~466Ò½”òóó‘ŸŸ—ËE 'Ñh”¡¯5¯Ûå”T²€sÆïOëÑ?  Êib«"jµ2™ 2™ óóóhmmÇqÐëõ°Z­P*•H$p»ÝqÏÛ“Àjµ‚çyp‡š¿H$Ïó°X, „`jj r¹ …V«&“ Ç¡³³“Þ—çyhµZH¥RX,hµZ@]]MÓÔÔŽã`2™——©T …BñÒ¿—œ³c«œ3XÀ9o7o¼µX,0›Í!ÃÃÃÈÉÉÙl¦ãÔôô4!¨¬¬Daa!ûÆÛ •g·Û™ñÁŒÆ7>d2¬Vë’cvv6Îø())!óó󘙙H$BqqqœˆÁ`€B¡Xb|D"B077©TwÏó°ÙlôÿF£*• sssôœÍfƒD"Á³gÏè5‰3334Éd‚R©¤ùˆÅb”——Óß'&& ‰˜ñÁŽ„0>æçç©UŒH$B¦¦¦Öýã{vv–ÌÍÍ­:ýÌÌ ™ŸŸ'sssdvvvÉï‹•®˜ñÁŒDo#‘êëë¡V«ÁóŸéééHOOÇž={PPP°ìýív;t:öìÙƒO?ýÀÿ¬pĈ­h<¾¥¥@¹¹¹qé.^¼ˆ´´4üä'?Áõë×—ä;88ˆ””ddd 99ƒƒƒ4¯`×®]köža0Ìø`0 Â/~ñ ˆsƒ€Gã¸e÷ Y r¹R©<ÏÓCœ>}:nåãuž/ö!Ã`$:uuup:p»ÝÈËËÃää$nÞ¼‰p8Œ±±1ŒŽŽb||½½½q×õöö¢¸¸˜˜˜@OOuz¾/ºÝnŒŽŽ¢®®~¿@ Îe2vÝóçóóóé>A2™,.]4Åðð0Ün7jkk—ä{ôèQܹsN§÷îÝCnn.¢Ñ(ü~?òòò0??¿!’2 f|0Œ-„Á` *Wýýýðù|hkkC}}=Š‹‹_içcŽã088ˆÞÞ^7ÒÜÜŒO>ù>Ÿ###8qâ¼^/T*Õªî©R©`6›QRRBŸïâÅ‹a•ÇHhbî†yyyHNNƉ' •J¡V«188¿ß‚‚Àï÷/ió§N‚H$Â¥K—››KWLžÇb±XXåÔjµxòäÉš<¿Éd°°Côó+>Ÿ"‘ˆN^¨T*H$j¼ìÝ»—É™26œP(´d}§Ãz!ƒÁxml¯ô±“È]Ì_|²²2˜Íf‚©TŠ’’|ôÑG4 ÏóKb>d2YÜÌeii)*++‘““ƒgÏžáÂ… à8uuu(//Fܽ{—~xÅVH#‘HâbPîÞ½‹³gÏÂd2!Â`0 ²²rÕ«' Æf"‹éŽÊ‹?‚bm¼¦¦§NB?êëë1>>Žã••E÷ˉF£KúIYYü~? qêÔ©äÏïað|ŒÕëº0¾ùæ›+ND Ææó¼êÕF®Ì¬fåf%e¬ÌƒàóùpîÜ9 ;Â?xð`Ù´<ÏãðáïäRÇx}8ŽÃÇQQQúúzºª›}mnnFAA8ŽƒL&Ã7h|EFF ''Ç»omm-*++ÑØصZââbðCWWnß¾¡¡¡m¿ÄÔ® ƒ±îèõzÈåò¸Ýë c;ÒÒÒ²D‚Úëõbÿþý‰D8vìXÜF¸V«mmmhmm…ÕjE4Ezz:|>ŒF#®^½º­Ê‡ ãáõzF—hõ3‹ñù|P(¬0 ƶÆáp ££‚ `hhR©_ý5xžÇ©S§ V«ÑÝÝMÓ;v ‡^¯~ø!‚Á &''qáÂ|øá‡/Œ¥KT¶µÛûb0Ö¿ßK—.Ñͼjjj– eì,®_¿ì÷ûÑÝÝ ‰D‚ªª*V@ c[SSSƒ‚‚466B«Õ"??Ç¡³³:R©YYYT[,Ãb±P‘™L†êêj¤§§C*•"''*• Ñht[ìU³­ŽŽÖŒuäÒ¥K8þ<+FCCC‰Dðz½Éd¨¬¬D~~>Û]šÁ`l{”J%Ün÷’ó÷îÝ[Ñ`YLUUUÜdÍ©S§¶Mùlk·+»ÝÎzƒ±N„B! QÌñèÑ#$''#==:^¯“““HMME0D8†N§Ãøø8ü~?M—œœL—šsssqèÐ!$%%½pS³¡¡!9réééHKKÑ#G B¡8€ôôt¤¤¤àÀ]]](**®]»påÊ”••!55©©©ÈÍÍE4E(¡C‡––†äädܼyÀ‚â‘#G””„®®.ÖžãîÝ»p:˜˜˜@__‚Á FFF˜Ì.ƒÁ`0¶÷ʇÕj…ËåbµÌ`¬?†Z­^1 °Z­p:P*•èííÅéÓ§át:a±XpöìYˆÅb˜L&¨T*8p………ÈÏÏG @jj*>üðCFÜ¿ÿ…KÎ1ßÚ§OŸB,£¨¨·o߆\.‡V«¥³JiiiB8†ÇãÁüü<ÍkffÇáÒ¥K¸rå 4 jjj‡éóƒAH¥RBâvrf,åÃ?DCCJJJ R©˜ÛƒÁ`0ãcû’ŸŸ«ÕÊj™ÁX|>ß9Õçñx<ü1=7888þ<222‡ñÙgŸ\.ÄbqܤAl¥cïÞ½/õuI€ÉdBOOÎœ9‘H„Ë—/ÃëõÂëõÒôƒÇQeÏž=0™L8vìär9\.xžGAA½&¶!”^¯_x‰rL±üeœ9s°Ùl8|ø0T*+ƒÁ`ƃÁ`¼Â˃ã^:ëû°_ìËûw  R„>Ÿ …Ç¡¦¦†555¯½Ky8Çqøä“OÐÙÙ‰ÊÊJ?~@€¦‰å÷ïßÇää$z{{QPP@w….--Å|@ÓI¥R¶¢úܺu )))(**ÂÀÀ3Ú ÆŽ  À¦»ƒAˆD¢¸qo³`R» ãµP(˜œœŒ;÷»ßý>ŸJ¥¡Pˆ®’vïÞ `aõ±¿¿Ÿ ƒ±F\¿~ׯ_ßô稨¨X1F1ÓÝØ×6õ´Å‰F£G(‚\.gùŒ-þ}ûP\\L¥ÿÔj5šššÐÔÔDÓܽ{÷ïßGEE.^¼‰D‚»wïâÑ£GP(ÈÏÏ°àŠÕÛÛ‹7n ¬¬ iiià8v»2™ jµ:nÖ¨±±¿øÅ/–ÌàH¥R! "''ÙÙÙøàƒPTT‡Ã¥R‰’’„ÃḞçQYY‰ŒŒ pFƒ“'O"ââÅ‹Ðét€ÚÚZ¨Õj(•Ê¸ëÛÛÛñöÛo¿ö*M¢òüÆWÙÙÙÈÎÎ~aúsçÎáܹs¬ól1är9,lŽÆ䲌íÇ矾âïÁ`.—k‰êVÂByíc³¯‘H„Îð-ù}jjjyëŽã ×ëát:iZ§Ó qç–;ššš–ÈR*•J8Ž5ù›¶òÁXŸþ´Öý£´´ÝÝÝÞ>JJJðìÙ³¸sN§ƒaSÚk]]&&&6´l¥û°cÝ꙼.&“‰ÌÏÏB)--%ííí„BH}}=)--%ÕÕÕ„çyÒÓÓC, q8D&“¥RI™˜˜ F£‘ b4ÉÄÄD\N§“”——Óÿ÷ôô††233C, ÑjµD«Õ’žžú£££4½Åb¡çKJJÏóKò¨¯¯'†h4RYYI"‘ÉÉÉ!³³³„Bª««ISS!„ÑÑQR]]MêêêˆÍf#Z­–¨T*ÒÜÜL6’ÿ®·-ßÿÙx»1ã­ÍfƒÍf!µµµP©TÐjµP(FCChúòòrtvv¢§§*• ƒ2™ ÍÍÍ „ ººjµjµ999ˆD"xöìÌf3Ôj5T*Z[[áv»a±X —Ëa4é}Ýn7²²² ×ë¡V«a±X‰D••‰D‚ÒÒÒíñ·ŒžžˆD"¨T*”””¼Ðø¨¬¬„Óé„Óé„Ãá@SS”J%Äb1ž>}ºj㣧§PRR‚±±1LMMa``F£Ç-k1ãcg¿ [[[Q[[KÛÙFôØ o£ÛÇrãf¯Ræëõñ±õÏŽ­k|dee‘¾¾>2??Od2ÉÊÊ"„R\\L:;;‰Á` V«•<}ú”LLLžç !„X­Vb·Û !„¨Õjâp8¨¡¡V«ãò˜››#R©”ÌÍÍB1›ÍÄét‹ÅBjkk !„LLL™LFfffˆÁ` N§“^ËÓ`0’’òôéS‰DèïÝÝÝD£Ñ¹¹9‰DˆÙl&¤°°SJ¥’ jˆ477«ÕJ, ‰D"dllŒˆÅbf|°ñvÓgÏžÁd2!‰PC¤¤¤P*• „`~~2™ sss°X,ô}>:: ¥R‰ùùyH$z›Í†§OŸ¢¼¼••• „`ffN§b±333˜ŸŸ‡Õj…Ýn‡Óé¤yB`±XÐØ؈©©© CYÌÇ*±ÛíÐjµ°X,hmm}áîéJ¥ƒƒ™™™8uêîß¿p8üJ›vwwC¥RáÆP©Tàyz½÷î݃H$BKK˲×-翘ØÞ+!²þô‹—æ[ÉÉITWWcÏž=HMMÅ•+W–OXK$ >ÿüó —›]ÎýÐ`0PÃ~£Ù î›QÿŒ­‹ÙlF__ž( ŒŒŒP—W™L†ááa€J¥‚L&[ÒÇ+Cfee½pÿ¬õ‚«  ¢»»9998~ü8B¡Ð ?þ_dÄ^æ«…ã8%×H$8NºÓ¥ßïÇîÝ»ÑÑÑÝ»wãÝwßÅ»ï¾'m ,:t?úÑðî»ï"99½½½qiFFFžžŽýèGxçw––Fd4ÅéÓ§ñÎ;ïàý÷ßÇ;＃¢¢¢aŒÍÅãñl؇¨T*eÊE;¸þ[“Ç£§§}}}0™LÐh4¸yó&]‰€·ß~{ÅI(±XŒ}ûöÑãáÇKâ¬òóóÑÚÚŠ¶¶6äååÅa+±ƒ O¶øžÑh™™™Ä£Gèd_GGÄbqœƒ±Y´´´P5Ç^¯û÷ï‡H$±cÇPRRB³Z­hkkCkk+¬V+¢Ñ(ÒÓÓáóù`4ãâë>|ˆû÷ïC¡P ¨¨mmm´¿,þžûî»ï–XX®/nôξVÙˆ8ŽÃ±cÇ •Ja4ÑÜܼê­îc†ÊË6d[Ì™3gÐÚÚŠäädäääÀh4â—¿ü%xž‡F£‰{9û|>£¹¹DGG !‘HpêÔ)‚€ŒŒ Èd2¸ÝnH$´µµ!++ N§z½@0 Çq¨¨¨À055…O?ýøâ‹/Àó<¾þúkX,ð<Ï6 Û`^¤F±œükRR<ýùˆ;vl[‰l)ÃÍl›Qÿ«Ùëe3'b¯ Kâç¶;R©‰ÝÝݨªª‚ÏçƒÍfCmmíª>úyž§ýÉ`0`||'Nœ 3ª1ôz=ü~?š››ñÅ_Ðsmmm8uêñ«_ý ÝÝÝøæ›o`0V¥¨e0ÐÔÔ„“'OR ƒÁ@] ›ššàv»!‰P^^Žâââï …è„àv[‡Ã`0ˆ3gÎ`hhf³_ý5xž§ße—.]¢Æò±cÇP__tcÝÉÉI\¸pb±—.]¢ïµÔÔTŒáüùót5Òh4âöíÛÐjµðz½8tèìvû Ÿ¯¯¯~¿2™ ííí´ï„B¡%FÂ’è1*• ‹%熇‡—Ä|èõzX­VzÄb4Ôj5æçç_)à|bbyyyq3Mjµ­­­Kò3Å«Õ …BBš››ÁqfffâÒFê¯o³Ù ‹© !ÓÓÓÈÊÊÂØØ ©¿aìw‡ÃWÌ5±(//_“þ±YÇôô4õq]зCb¶Dý¿ìú‰‰ Ô××oJÅü›ív;¬VëŠi+++1==½­b>!¤±±‘ÆzÄâ:bÚ‹…¸ÝnB!ÓÓÓD¯×Bimm%‰„´¶¶·ÛMT*ÑjµD¡PÐøç©­­¥q„233CŒF#Ñh4D¥RÑ·ÛMxž'jµšÓ<?Ëó”——¥RIT*),,¤Aô­­­4ÏÙÙYÂóÀ·ß~ ³ÙŒ¼¼<*]ÓI^‚S"‘À`0àäÉ“Ôˆp¹\ÈÈÈ ²«™…ÁãÇÑÞÞǃºº:\¸pæëv»¡Õjé5·oßFaa!ž={†ŠŠ tvv¢´´t‰åëõz1;;‹C‡! Q¹Åçéïï‡Éd‚X,†Éd‚ÑhÄáÇ×m61))‰½õ^€Íf{áÊÇË6¿Ûµk²³³qá¨ÕêÜ?6—Ë…Ë—/oZœÇf’””´¤¬¦þßzë-üÝßýÎ;·&õÿ²ë[ZZàr¹^*ñ¸À`0P9ç•Ø½{7œNç–[ÅIJJ"‰Ð?+** R©VUÖ;¥B’6²ÿ¯Ç˜ÂÀ¶4övÌØ™È++-ãŠD"*÷[°Ûí/½çËV>æççÑÙÙ¹¬jM$Ñh„T*Ë÷ù´±å¶ééiX­VÈd²¸™Å!„úÍ®ôÜ1+:¶d-‰â¬t6³³¹ýi¥äÝwßEuuõ’Õ¯Ò?†‡‡¡V«a0 R©000@WÚbR‚ƒ333˜žž†Ñh„Ñh„J¥‚ÑhDuu5´Z- Ž[<+¤V«¡ÕjñìÙ3tvv"//z½J¥’ª…Äöþ(--“*Œ)Ìi4˜Íæ¸Õ¼í´òñ*õÿÞ{ï­yý/¾¾¡¡¥¥¥q3}„:SWW‡ÙÙY{s™‰ÉJfeeXØ¿%6¹¡V«é¬œ^¯‡L&C___œ\eÌlnn‘HV«•¶=µZ »ÝŽÎÎN:»X__•J½^žçáv»QWWGƒ8gffhûÑjµ0˜MØ•õ&‰½^OT*]‘`0µ+öýºµÍT†dR»«8æçç!‹—Õ¶®ñ¡T*ÑÔÔç.·Vý£°°~@ºÝn*Õ'—Ë©QÞØ؈ÂÂBLMMA$Q×¥RIûNìÃÕétƵù¬¬,´··Ãn·C¡PпA£Ñ``` îs»Šå38ÊËË—ôd|¬gý/¾Þf³A«Õ"‰ ‰€çyLLLĹ<S׸¹¹9( LMMÁjµ¢°°¾“€ÿÙKI£ÑÐkš››QXXˆ¨Õj*5Y^^ŽÚÚZ´¶¶R7ÒØý»]ÍÍÍÁd2Ѷk›„ð<©©)<}ú …‚¦ihh@qq13>V`tt”Ìø`Æ;^é`ç+ÐÑÑp8ŒãÇ/ùM«ÕÒ}­wì‹ÅÈÉÉACCÔj5Ž;Fóz½hmm…ÉdŠ»&l,(455Áb±X0lhhÀíÛ·qòäIšfÿþýP*•xøð!Ìf3Z[[ár¹¨+Ø£GP__ììlœ>}šºeq¹\F³áòlŒÕa4QZZJÛÄz““‹Å‚îîn˜L&?~=Çq(++°¼æõz,ÈÿÅ‚ëd2~þóŸXpM …B0 à8W®\×ëÅàà ×ëõ4xX¥R½PÚ÷É“'0 ÔÅñù¸Yý¯ož± Ežç—(õ¹\.( ÄÕW¬~Ðw‹J¥¢ï˜»ëÏ~ö3 âñãLjF£(**° øÇqü~?m/b±F£qÉ{õƸyó&|>úúúâ\U?ÏÙ³g,wú|>öBYWRa0 `Ô®’’’6r>6ãºX]j1………(//‡ËåZs_áÏ>û ~¿‹%%%P©T…Bðxúè#:°—––¢°°ÝÝÝày‡‚ àÆ”:;;qàÀäää€ã8´··SWŠ>úHKKƒÁ`€ÏçCOOY/Ú"ˆD"X­VêÏ¿ÞdffbjjŠÆ"µ¶¶¢¼¼*• 555ÿó’ùoyÌ—ÑÖÖ†††TVV";;ûµÞÏ燷•²×V©ÿåÊþe”””àƒ> ÿ—J¥èëë{åûèõz\¸p!îo¿råÊŠÒß>Ÿéé騮®F~~>T*†††–ä­T*iûF£«’ªd0Œ1>>Žp8 •JE'Æ¢Ñ(Àiè_Ìää$ÔjõéëDaÝ÷ù‰DK´Ž«ÕŠ;wî¼ð÷üü|ØívˆÅbH¥RØívìÝ»÷¥÷U*•°Ûí+ÊýI$ Àáp °°n2ØÞÞŽÑÑѸ vTZ”——cxx8.͵k×000¥R APZZŠ‰‰‰¸²{÷³‰b±¸ví@£Ñ`tt‹¡P*• ÃÃë–f¬?.\À矾až¹¹¹hiiÁáÇQUU…@ €}ûöaddáp<Ï£¥¥«ºßÐÐL&²³³!‹áp8^zM8ŽÛ/`ß¾}p¹\ðûýˆF£8{ö,¾úê+Vÿ›„X,F(‚ 0›Íèîî†\.‡H$‚N§{­½G<‡ÃŽãÀó<._¾ŒëׯÃl6Ãn·## §§'¬ä©S§ R©–ü ñË_þ###¶_&iÊ`0^¯×‹””àòåËسg݃Íï÷#==}É5/:3:t:Ž=Š?þ{öìÁ§Ÿ~še³!+~¿ËjÁ¯ÄË”;¤Ri\šÕ*}Èd²U§ÍÌ̤;¶®Ä›o¾‰S§N­h èõzêÞ°ÒÀþ"W …B7ÛÈØÙ\½zEEEhooÇq¸{÷.d2ìv; èÇaÌ€]ìâ õ¥R‰ììl ##r¹%%%à8R©4ÎPW*•H$P(‹Å(**¢nˆ2™ ÍÍÍ8zô(€…}»-2Ö–˜‹æóï÷Xý*•JÔ××ãâÅ‹øè£PQQN¨««ƒZ­ŽSŒ~/×Nb÷S«Õ¨­­Å‘#GèÄHUUÄb1ÆÇÇ‘‘‘AUùb÷«T*¤§§C*•Âd2a||P\\ŒÜÜ\  ¹¹™º†ñ6$ ÜívC¸·H¥ÒMÙÀ‹Á`0 ƒ±<Ë `,±àyž‡FSSªªª,(E^¿~À‚Ë©T*‹oLtÞØLÞx?f-‘Á`0 ƒ±­ÑëõK”ü ¿¿ÙØÛç•õb1“<ÏC.—lja,æôéÓ é]´aŽ~jµ£££¯þ€,æcGÂ|P_Ìv‹ùƒ‰DºtS>Šý{1r¹|ËKƒo…˜A–ì呈Â"/#CH¥Òͨgó‘ ã‹ù`ãíþýû¡P(ð«_ý ‰½½½ÈËË×_~ ©TŠÝ»wc``z½§OŸ†D"Á©S§‘‘±lÐynn.8ŽÃµk× •JÑÒÒ‚òòrŒ%œëý•Ñ7ß|³*éLƒ±³¨¨¨@WW׆晞ž¿ß¿ßÝ»w£  €ï¿ÿ>.]ºÄ*æ% !55•–[nn.Þÿ}ܾ}{Ë?{nnîK ŽÓ§OºººV-Í`01îß¿‰DN‡Ý»w£±±_~ù%4 D"²²²ð駟B§Ó!R—«qëÖ-Èd2z¿ööv<|ø0!c~7ÔÜýéOŠÉÉÉWšUd+;6ób¶ÛÊGAA êå§×‚˜B\ìß‹Ë `Ïž=˜˜˜Xq£§Íòár¹pùòeZŽ1ƒdÿþý˜››Ûòï—•þnŸÏ÷ÂÙÇ ~N¶ò‘ ã[ù`ãíëàõz‘››ûZžB‰Ä™Y x©eÇ`0ƒ`0ˆ@§Ó!99‡B4Åõë×QTTDggŠŠŠ,l”žžŽôôtìÙ³‡žQQQç»ZPP€›7o"##ƒ{öìÁíÛ·‘‘Aónii¡÷(((@RR_ëïZ¼-ãÕH$´Ü¼^/ÒÒÒžžŽäädôööXXu(((Xr>==GŽÁ®]»pöìY¤¤¤ 55§OŸF4…ÇãAJJ 222œœŒþþ~ ²–ÉÉÉÐétÈÈÈ@ €ßïÇþýû‘‘‘´´4ìÙ³>Ÿeeeô9b²–iiiôƒ(++C @nnnÜÊGKK öìÙNN¯× HKKÑ#G––†Ý»wÓóŒG8Æ[o½õ'VŒWeppG¥ûV1Ö]»v¡··„U1Ëóuz=;6ç`¬<³–åüº×¶··£±±‘þ_­VÃívÃf³Ád2!‰ ‰@*•bzzõõõhoo!óóóH$xöì¬V+ìv;úúú ×ëAÁìì,xžË¯¹¹ƒ‘HV«•æ=77žç1== «ÕŠÒÒRš÷‹ž=¶‘SlfÛjµÒƒçy”——oùþ±Ù÷q:Éd´Ü, xžGSS!0èîî!333Ë嘟Ÿ‡Á`@}}=!xúô)är9"‘½v~~ííí0´srrÐÚÚŠ’’444€‚ááa466bzzr¹333 „ ±±V«SSS‰D˜žž!ÅÅŨ««‹û»Ýn7JJJèßËgjjŠ¶?»Ý«ÕŠééiH¥RšÝn‡Ñh¤÷°§ t IDAT!6› ¥¥¥kR?IIIÑH$B‰ÃÓ§OÉŸýÙŸ¡ÿ³ñvcÇÛ—‘H„¾_¶û±áQ•‘HÙÙÙÛµ˜Á`$ÇŽãGpùòeŒÃçóQ%­VKÝ+U*¢Ñ(Î;‡®®.\¼x^¯‚  Ñû}øá‡(,,„ßïǃ——G{ôèàt:Áq\.Âá0FFFè*ÅW_}0¯0žŸŸo¿ý6› YYY¸zõ*«àU “ÉŸŸßÿþ÷°Ùl°ÙlÔ}ÎårA&“¡»»ÀB€zLßþðáÃ…B‘HDƒþ333!‰àr¹ …èêX €Ëå‚ÅbA^^zzz`6›qüøqô÷÷#ââÅ‹tæ9–L&£®s2™l‰RŒV«E4Å•+W099 —˳ټìßúÕW_Á`0PÿêãÇÓÍse2•Ìäy.—kMÊW$Íúý~Ùv äß®øý~¼õÖ[3¬$¯ Çq;f϶M‘tó7ƒý×ÝV»r3;‰+W®`ppÅÅÅÈÏÏlj'VLâÄ p‡¼¼<œ;w©©©KÒX­V´´´ »»wïÞŒ£¸¸>¤ŠCÇáÌ™3T]©ªª 2™ ===øñüÊKì=ôË_þ©©©Ðh48~ü8«ä— ‘HhÙýüç?GZZ>øàhµZZ/1ùÈšššeVA–5Ífs\X,†T*ÅÄÄž¶s¿c³2ŽD"0lƒ<#ABNN>· ñ"<N:…ÌÌL|ýõ×KdZ…»Ý‘H…B@ €#GŽàÎ;q+¥F£àyÇ!##Á`pÙËåòYécº©© %%%¯tcaÃf³¡  ‚ Àd2¡«« <ÏC¤¥¥ÑM²b1:ýýýH$KûÍf3zzzèoEEEhkkÉ'ÐÒÒ‚ƒ¢ªª @{÷î…ÇãA8ÏóhkkÃÙ³g_ú¼Á`F£ÇŽƒD"‰[±x~enïÞ½Âøø8àúõëëî›-Â7ãããaÖº‡ßüæ7 ‡Ãÿ/+ 6>&''ÓxÅääd$''crrò…×ܼyCCC„P¶3›*fÿÝwßáäÉ“¸víº»»ñWõW¬Å2 BUUÊÊÊÐÚÚ ¥R‰ÒÒR|÷ÝwKfkÔj5D"êëëQVV±X ­V‹’’„Ãa(•J:#ÛP)ærõõ×_C.—ÇIßšL&\½zeeeHKK444€çy(•JH$š¶±±ï½÷Þ’w­VKg«Ÿ_}ÍÌÌDqq1:::pîÜ9VÑ+jjµ:îÜ™3g022‚GáÖ­[(++ƒN§´¶¶Ò•«ññq¤§§C,ãÞ½{KêäàÁƒðz½Ø¿?­£3gÎ  ¢  ííí€öövÈd2Øívê¢%—ËñÙgŸÑ{ÆXÜ.‹‹‹‘žžŽ‡¢  Ëå(..† ËåÐjµ8zô(ÊËË¡T*!•JñÅ_Ðàw…B±l>2™lÍ\Š¿ÿþûû·nÝ2‹Y‹K îܹ#‚ÐÅJ‚¡ÕjãÔ+**péÒ%ºªÿ<íííøàƒXÁm$IIIÈÎÎÆÄÄD|P Ç­8ÊÎYÀùNc+.þÐc~~ …Ïž=[“ûUVVÒ€ãíÖ?µþ œN'{Ï­®~Ä»ví ÏÍͱHî 6ÿñüûDéÿl¼]¿ñÖétÂ`0Äëìì„ÑhDyy9Z[[éy«ÕŠââb:¡“ÙÍË˃Z­†\.‡Ûí¦Â+J¥Z­ƒ333˜žž†Éd‚Ñh„Z­†R©ÄÔÔûÆ{UÞyç?~ÝÝÝø‹¿ø‹TˆÌø`Æ{&FÿèééT*¥ŠDkqlõ03>Øñ²ã­·ÞzØÜÜüöi¿õ©®®ž‰DMÌø`ã­Óé„R©„Ýn‡ÝnGSS”J%Ñ××G “ÙÙY( D"‘¸w#ôôô€‚¦¦&RuÀ˜"VSSrrr055ŽãðôéSªìW[[ËÔ®^•¹¹9´µµ¡­­ o¿ý6ü~?XÀƒ±½9xð &&&¨kÎZÀÞ[ÅnŒ—ó§?ýé㊊ M~~þÿöª*nŒ# áÿñ¿¡ž•#Ö&bqd‰õõõ8xð €q•˜²cNNβ¢±´|ðÚÛÛñäÉ“8ŽcÇŽ¡ººÀ‚»©B¡°àú¹Õ÷©Úòo²ï¿ÿ^¯z½žµdc›³–†ƒ± „¸8Žû~Y^^þ+‘­IMMÍ"‘ÈÿE™d¥Á¥RùBQ¥ÂÂB´´´ÀápàÖ­[«û`NÅo±â^¢ñÆVÀH$·ë1ƒÁ`übjE F¢ðÝwßUýÃ?üÛLmë¾SnÞ¼ù}$ù?Yi0VCLÙ1Æ TüáxáªÅ¾}ûàr¹àóù,(î%êvo$ÂCºÝî%›C1 °  ò*<~ü˜Mh0 Bˆg~~¾~ïÞ½3l,ÜZƒAÆÿúöÛoOB˜uÈ°¼àbbÊŽV«•ž³Z­(--…Çã‰3*b÷’ËåhnnFnn.t:ü~?nݺ‘H´DÙo«»'%B%þèG?Âýû÷‘™™ùê`R ¨J@’’’X!¼øCdMË9ÑûÇîÝ»155µêôÑhiiip»Ýë¶9ÜFôµ¨7ö~L,Þ|óÍÖƒþùŸÿù/Yil>Ñhz½þÙ¯ýëÏ"‘HU¢õ6Þnìxû|ÛINNÆððpœ<üN!!V>æççQYYÉzƒ±…‡Ã8pàRSS¡ÓépäÈ !ãèÑ£HMMEJJ >þøc KÄ'Nœ@ZZvïÞ²²2@ @FFt:RRRÐÕµ ‘Ÿ››‹'N )) ƒƒƒ8}ú4ÒÒÒ Ó霜 ŸÏ‡²²2äææ"âìÙ³HIIAjj*rss!ºººpâÄ ¼óÎ;8}ú48Žƒ^¯§ù0‰Â÷ß_üðáÃÿuðàA¶²É„B! †gcccƒÑh´†•cµôööâ'?ù ¬VëŽ4<Æø€o¾ù‡ƒµZc‹pýúu¨T*ŒŽŽâþýû‚ øä“OÀóÿãÿøB¢¬T«åàÁƒCUUÕŽ-ƒ„1>æççQ\\¼ååÃŒ‚Ëå¢R€2™Œú¨º\.Œ£  EEE‡Ãxòä Àh4Äb1¤R)B¡FFFÐ×ׇ‚‚œ={‚ à׿þ5MÏqT*Îœ9ƒ?þEEEèìì\ær¹â$ - 50ôz=Äb1u³úó?ÿsLN2QFb ßÿ}ÞäääŸýìgÿ_CCC˜‹ƒ ¸té’ššþÏÿüÏÒH$ržŒ×!&—»SI(®@ €ªª*êÆÁ`06‘H„o¿ý6n`äóòò°wï^ ¾­R©ׯ_aŒÅ™3gh€\UU}1¿ýöÛÔ°8}ú4ª««‘™™ ¹\¾ìó,þæÇöG`lC#äJRRRoeee]mmmÚÕ«Wß9zôè.±XÌ g ƒ¸{÷n¤ººúO‘Hä 5LR—±Ú¶3>>•Jµj)ùP(Žã ‹!FFF T*·•ý‰ô°ß~û-®]»†ù—a-šÁØdrrrÐØØH7RŠ¹E ´··C.—C"‘àÈ‘#+ÊÛFôöö‚çypN‡`0—æ׿þ54 Ž?žçÑÙÙI ‡Ã‡Ã0™L°ÛíÔ½«±±&“iÙ<¿ùæ¨T*V‰ŒD7@<‚ ˜~ÿûßg—””¸Þ{ï½?êtºà­[·"ƒƒƒðù|Ì[à>Ÿ.— ßÿâ¿xö—ù—üû¿ÿûÞÿú¯ÿÒÿñ<Î ÆjˆÅ)^¾|iii(((XU¼~ý:I¥›¦§§£··‡^ñÚššÿÑ/èëëC$Çq(--ÅÕ«W·Eùüÿ¶hæ¦\ ã]IEND®B`‚stone-3.3.1/example/000077500000000000000000000000001417406541500143105ustar00rootroot00000000000000stone-3.3.1/example/__init__.py000066400000000000000000000000001417406541500164070ustar00rootroot00000000000000stone-3.3.1/example/backend/000077500000000000000000000000001417406541500156775ustar00rootroot00000000000000stone-3.3.1/example/backend/__init__.py000066400000000000000000000000001417406541500177760ustar00rootroot00000000000000stone-3.3.1/example/backend/ex1/000077500000000000000000000000001417406541500163745ustar00rootroot00000000000000stone-3.3.1/example/backend/ex1/__init__.py000066400000000000000000000000001417406541500204730ustar00rootroot00000000000000stone-3.3.1/example/backend/ex1/ex1.stoneg.py000066400000000000000000000004701417406541500207420ustar00rootroot00000000000000from stone.backend import CodeBackend class ExampleBackend(CodeBackend): def generate(self, api): """Generates a file that lists each namespace.""" with self.output_to_relative_path('ex1.out'): for namespace in api.namespaces.values(): self.emit(namespace.name) stone-3.3.1/example/backend/ex2/000077500000000000000000000000001417406541500163755ustar00rootroot00000000000000stone-3.3.1/example/backend/ex2/__init__.py000066400000000000000000000000001417406541500204740ustar00rootroot00000000000000stone-3.3.1/example/backend/ex2/ex2.stoneg.py000066400000000000000000000012001417406541500207340ustar00rootroot00000000000000from stone.backend import CodeBackend class ExamplePythonBackend(CodeBackend): def generate(self, api): """Generates a module for each namespace.""" for namespace in api.namespaces.values(): # One module per namespace is created. The module takes the name # of the namespace. with self.output_to_relative_path('{}.py'.format(namespace.name)): self._generate_namespace_module(namespace) def _generate_namespace_module(self, namespace): # pylint: disable=unused-argument self.emit('def noop():') with self.indent(): self.emit('pass') stone-3.3.1/example/backend/ex3/000077500000000000000000000000001417406541500163765ustar00rootroot00000000000000stone-3.3.1/example/backend/ex3/__init__.py000066400000000000000000000000001417406541500204750ustar00rootroot00000000000000stone-3.3.1/example/backend/ex3/ex3.stoneg.py000066400000000000000000000040241417406541500207450ustar00rootroot00000000000000from stone.ir import is_struct_type from stone.backend import CodeBackend from stone.backends.python_helpers import ( fmt_class, fmt_var, ) class ExamplePythonBackend(CodeBackend): def generate(self, api): """Generates a module for each namespace.""" for namespace in api.namespaces.values(): # One module per namespace is created. The module takes the name # of the namespace. with self.output_to_relative_path('{}.py'.format(namespace.name)): self._generate_namespace_module(namespace) def _generate_namespace_module(self, namespace): for data_type in namespace.linearize_data_types(): if not is_struct_type(data_type): # Only handle user-defined structs (avoid unions and primitives) continue # Define a class for each struct class_def = 'class {}(object):'.format(fmt_class(data_type.name)) self.emit(class_def) with self.indent(): if data_type.doc: self.emit('"""') self.emit_wrapped_text(data_type.doc) self.emit('"""') self.emit() # Define constructor to take each field args = ['self'] for field in data_type.fields: args.append(fmt_var(field.name)) self.generate_multiline_list(args, 'def __init__', ':') with self.indent(): if data_type.fields: self.emit() # Body of init should assign all init vars for field in data_type.fields: if field.doc: self.emit_wrapped_text(field.doc, '# ', '# ') member_name = fmt_var(field.name) self.emit('self.{0} = {0}'.format(member_name)) else: self.emit('pass') self.emit() stone-3.3.1/example/backend/unstone/000077500000000000000000000000001417406541500173725ustar00rootroot00000000000000stone-3.3.1/example/backend/unstone/__init__.py000066400000000000000000000000001417406541500214710ustar00rootroot00000000000000stone-3.3.1/example/backend/unstone/unstone.stoneg.py000066400000000000000000000134421417406541500227410ustar00rootroot00000000000000"""Example backend that outputs a Stone file equivalent to the input file. Current limitations: - Whitespace is not reproduced exactly (this may be a feature) - Order of definitions is lost - Comments are lost - Aliases are lost (they are expanded in-line) - Docstrings are reformatted """ from __future__ import absolute_import, division, print_function, unicode_literals import six from stone.frontend.ast import AstTypeRef from stone.ir import DataType from stone.ir import List, String, Timestamp from stone.ir import Struct, Union, Void from stone.ir.data_types import _BoundedInteger, _BoundedFloat from stone.backend import CodeBackend class UnstoneBackend(CodeBackend): """Main class. The Stone CLI finds this class through introspection.""" def generate(self, api): """Main code generator entry point.""" # Create a file for each namespace. for namespace in api.namespaces.values(): with self.output_to_relative_path('%s.stone' % namespace.name): # Output a namespace header. self.emit('namespace %s' % namespace.name) # Output all data type (struct and union) definitions. for data_type in namespace.linearize_data_types(): self.generate_data_type(data_type) # Output all route definitions. for route in namespace.routes: self.generate_route(route) def generate_data_type(self, data_type): """Output a data type definition (a struct or union).""" if isinstance(data_type, Struct): # Output a struct definition. self.emit('') self.emit('struct %s' % data_type.name) with self.indent(): if data_type.doc is not None: self.emit(self.format_string(data_type.doc)) for field in data_type.fields: type_repr = self.format_data_type(field.data_type) if not field.has_default: self.emit('%s %s' % (field.name, type_repr)) else: self.emit('%s %s = %s' % (field.name, type_repr, self.format_value(field.default))) if field.doc is not None: with self.indent(): self.emit(self.format_value(field.doc)) elif isinstance(data_type, Union): # Output a union definition. self.emit('') self.emit('union %s' % data_type.name) with self.indent(): if data_type.doc is not None: self.emit(self.format_string(data_type.doc)) for field in data_type.fields: name = field.name # Add a star for a catch-all field. # (There are two ways to recognize these.) if field.catch_all or field is data_type.catch_all_field: name += '*' if isinstance(field.data_type, Void): self.emit('%s' % (name)) else: type_repr = self.format_data_type(field.data_type) self.emit('%s %s' % (name, type_repr)) if field.doc is not None: with self.indent(): self.emit(self.format_value(field.doc)) else: # Don't know what this is. self.emit('') self.emit('# ??? %s' % repr(data_type)) def generate_route(self, route): """Output a route definition.""" self.emit('') self.emit('route %s (%s, %s, %s)' % ( route.name, self.format_data_type(route.arg_data_type), self.format_data_type(route.result_data_type), self.format_data_type(route.error_data_type) )) # Output the docstring. with self.indent(): if route.doc is not None: self.emit(self.format_string(route.doc)) # Table describing data types with parameters. _data_type_map = [ (_BoundedInteger, ['min_value', 'max_value']), (_BoundedFloat, ['min_value', 'max_value']), (List, ['data_type', 'min_items', 'max_items']), (String, ['min_length', 'max_length', 'pattern']), (Timestamp, ['format']), ] def format_data_type(self, data_type): """Helper function to format a data type. This returns the name if it's a struct or union, otherwise (i.e. for primitive types) it renders the name and the parameters. """ s = data_type.name for type_class, key_list in self._data_type_map: if isinstance(data_type, type_class): args = [] for key in key_list: val = getattr(data_type, key) if val is not None: if isinstance(val, AstTypeRef): sval = val.name elif isinstance(val, DataType): sval = self.format_data_type(val) else: sval = self.format_value(val) args.append(key + '=' + sval) if args: s += '(' + ', '.join(args) + ')' break if data_type.nullable: s += '?' return s def format_value(self, val): """Helper function to format a value.""" if isinstance(val, six.text_type): return self.format_string(val) else: return six.text_type(val) def format_string(self, val): """Helper function to format a string.""" return '"' + val.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\n\n') + '"' stone-3.3.1/ez_setup.py000066400000000000000000000240741417406541500150740ustar00rootroot00000000000000#!/usr/bin/env python """Bootstrap setuptools installation To use setuptools in your package's setup.py, include this file in the same directory and add this to the top of your setup.py:: from ez_setup import use_setuptools use_setuptools() To require a specific version of setuptools, set a download mirror, or use an alternate download directory, simply supply the appropriate options to ``use_setuptools()``. This file can also be run as a script to install or upgrade setuptools. """ import os import shutil import sys import tempfile import zipfile import optparse import subprocess import platform import textwrap import contextlib from distutils import log try: from site import USER_SITE except ImportError: USER_SITE = None DEFAULT_VERSION = "3.1" DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/" def _python_cmd(*args): """ Return True if the command succeeded. """ args = (sys.executable,) + args return subprocess.call(args) == 0 def _install(archive_filename, install_args=()): with archive_context(archive_filename): # installing log.warn('Installing Setuptools') if not _python_cmd('setup.py', 'install', *install_args): log.warn('Something went wrong during the installation.') log.warn('See the error message above.') # exitcode will be 2 return 2 def _build_egg(egg, archive_filename, to_dir): with archive_context(archive_filename): # building an egg log.warn('Building a Setuptools egg in %s', to_dir) _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir) # returning the result log.warn(egg) if not os.path.exists(egg): raise IOError('Could not build the egg.') def get_zip_class(): """ Supplement ZipFile class to support context manager for Python 2.6 """ class ContextualZipFile(zipfile.ZipFile): def __enter__(self): return self def __exit__(self, type, value, traceback): self.close return zipfile.ZipFile if hasattr(zipfile.ZipFile, '__exit__') else \ ContextualZipFile @contextlib.contextmanager def archive_context(filename): # extracting the archive tmpdir = tempfile.mkdtemp() log.warn('Extracting in %s', tmpdir) old_wd = os.getcwd() try: os.chdir(tmpdir) with get_zip_class()(filename) as archive: archive.extractall() # going in the directory subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0]) os.chdir(subdir) log.warn('Now working in %s', subdir) yield finally: os.chdir(old_wd) shutil.rmtree(tmpdir) def _do_download(version, download_base, to_dir, download_delay): egg = os.path.join(to_dir, 'setuptools-%s-py%d.%d.egg' % (version, sys.version_info[0], sys.version_info[1])) if not os.path.exists(egg): archive = download_setuptools(version, download_base, to_dir, download_delay) _build_egg(egg, archive, to_dir) sys.path.insert(0, egg) # Remove previously-imported pkg_resources if present (see # https://bitbucket.org/pypa/setuptools/pull-request/7/ for details). if 'pkg_resources' in sys.modules: del sys.modules['pkg_resources'] import setuptools setuptools.bootstrap_install_from = egg def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, download_delay=15): to_dir = os.path.abspath(to_dir) rep_modules = 'pkg_resources', 'setuptools' imported = set(sys.modules).intersection(rep_modules) try: import pkg_resources except ImportError: return _do_download(version, download_base, to_dir, download_delay) try: pkg_resources.require("setuptools>=" + version) return except pkg_resources.DistributionNotFound: return _do_download(version, download_base, to_dir, download_delay) except pkg_resources.VersionConflict as VC_err: if imported: msg = textwrap.dedent(""" The required version of setuptools (>={version}) is not available, and can't be installed while this script is running. Please install a more recent version first, using 'easy_install -U setuptools'. (Currently using {VC_err.args[0]!r}) """).format(VC_err=VC_err, version=version) sys.stderr.write(msg) sys.exit(2) # otherwise, reload ok del pkg_resources, sys.modules['pkg_resources'] return _do_download(version, download_base, to_dir, download_delay) def _clean_check(cmd, target): """ Run the command to download target. If the command fails, clean up before re-raising the error. """ try: subprocess.check_call(cmd) except subprocess.CalledProcessError: if os.access(target, os.F_OK): os.unlink(target) raise def download_file_powershell(url, target): """ Download the file at url to target using Powershell (which will validate trust). Raise an exception if the command cannot complete. """ target = os.path.abspath(target) cmd = [ 'powershell', '-Command', "(new-object System.Net.WebClient).DownloadFile(%(url)r, %(target)r)" % vars(), ] _clean_check(cmd, target) def has_powershell(): if platform.system() != 'Windows': return False cmd = ['powershell', '-Command', 'echo test'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_powershell.viable = has_powershell def download_file_curl(url, target): cmd = ['curl', url, '--silent', '--output', target] _clean_check(cmd, target) def has_curl(): cmd = ['curl', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_curl.viable = has_curl def download_file_wget(url, target): cmd = ['wget', url, '--quiet', '--output-document', target] _clean_check(cmd, target) def has_wget(): cmd = ['wget', '--version'] devnull = open(os.path.devnull, 'wb') try: try: subprocess.check_call(cmd, stdout=devnull, stderr=devnull) except: return False finally: devnull.close() return True download_file_wget.viable = has_wget def download_file_insecure(url, target): """ Use Python to download the file, even though it cannot authenticate the connection. """ try: from urllib.request import urlopen except ImportError: from urllib2 import urlopen src = dst = None try: src = urlopen(url) # Read/write all in one block, so we don't create a corrupt file # if the download is interrupted. data = src.read() dst = open(target, "wb") dst.write(data) finally: if src: src.close() if dst: dst.close() download_file_insecure.viable = lambda: True def get_best_downloader(): downloaders = [ download_file_powershell, download_file_curl, download_file_wget, download_file_insecure, ] for dl in downloaders: if dl.viable(): return dl def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL, to_dir=os.curdir, delay=15, downloader_factory=get_best_downloader): """ Download setuptools from a specified location and return its filename `version` should be a valid setuptools version number that is available as an egg for download under the `download_base` URL (which should end with a '/'). `to_dir` is the directory where the egg will be downloaded. `delay` is the number of seconds to pause before an actual download attempt. ``downloader_factory`` should be a function taking no arguments and returning a function for downloading a URL to a target. """ # making sure we use the absolute path to_dir = os.path.abspath(to_dir) zip_name = "setuptools-%s.zip" % version url = download_base + zip_name saveto = os.path.join(to_dir, zip_name) if not os.path.exists(saveto): # Avoid repeated downloads log.warn("Downloading %s", url) downloader = downloader_factory() downloader(url, saveto) return os.path.realpath(saveto) def _build_install_args(options): """ Build the arguments to 'python setup.py install' on the setuptools package """ return ['--user'] if options.user_install else [] def _parse_args(): """ Parse the command line for options """ parser = optparse.OptionParser() parser.add_option( '--user', dest='user_install', action='store_true', default=False, help='install in user site package (requires Python 2.6 or later)') parser.add_option( '--download-base', dest='download_base', metavar="URL", default=DEFAULT_URL, help='alternative URL from where to download the setuptools package') parser.add_option( '--insecure', dest='downloader_factory', action='store_const', const=lambda: download_file_insecure, default=get_best_downloader, help='Use internal, non-validating downloader' ) parser.add_option( '--version', help="Specify which version to download", default=DEFAULT_VERSION, ) options, args = parser.parse_args() # positional arguments are ignored return options def main(): """Install or upgrade setuptools and EasyInstall""" options = _parse_args() archive = download_setuptools( version=options.version, download_base=options.download_base, downloader_factory=options.downloader_factory, ) return _install(archive, _build_install_args(options)) if __name__ == '__main__': sys.exit(main()) stone-3.3.1/mypy-run.sh000077500000000000000000000004541417406541500150170ustar00rootroot00000000000000#!/bin/bash -eux EXCLUDE='(^example/|^ez_setup\.py$|^setup\.py$)' # Include all Python files registered in Git, that don't occur in $EXCLUDE. INCLUDE=$(git ls-files "$@" | grep '\.py$' | grep -Ev "$EXCLUDE" | tr '\n' '\0' | xargs -0 | cat) MYPY_CMD=mypy $MYPY_CMD $INCLUDE $MYPY_CMD --py2 $INCLUDE stone-3.3.1/mypy.ini000066400000000000000000000001551417406541500143550ustar00rootroot00000000000000[mypy] # follow_imports = False ignore_missing_imports = True show_none_errors = True strict_optional = True stone-3.3.1/requirements.txt000066400000000000000000000000261417406541500161370ustar00rootroot00000000000000ply>= 3.4 six>= 1.12.0stone-3.3.1/scripts/000077500000000000000000000000001417406541500143445ustar00rootroot00000000000000stone-3.3.1/scripts/release_note_generator.sh000077500000000000000000000007021417406541500214150ustar00rootroot00000000000000#!/bin/sh last_version=$(git tag --sort v:refname | tail -n 2 | head -n 1) echo "Getting commit history since $last_version" num_commits=$(git rev-list --count $last_version..HEAD) echo "Found $num_commits commits since last revision" git_log=$(git log -n $num_commits --pretty="format:* %s %n") linked_log=$(echo "Release Notes: \n\n$git_log" | sed -e 's/#\([0-9]*\)/[#\1](https:\/\/github.com\/dropbox\/stone\/pull\/\1)/g') echo "\n\n$linked_log" stone-3.3.1/scripts/update_version.sh000066400000000000000000000004451417406541500177320ustar00rootroot00000000000000#!/bin/sh if [ -z $1 ]; then echo "error: $0 needs a version number as argument."; exit 1 else set -ex NEW_VERSION=$1 git checkout main git reset --hard HEAD git tag "v${NEW_VERSION}" -m "${NEW_VERSION} release" git push origin git push origin --tags fistone-3.3.1/setup.cfg000066400000000000000000000002161417406541500144750ustar00rootroot00000000000000# See http://doc.pytest.org/en/latest/goodpractices.html#integrating-with-setuptools-python-setup-py-test-pytest-runner [aliases] test=pytest stone-3.3.1/setup.py000066400000000000000000000051301417406541500143660ustar00rootroot00000000000000# Don't import unicode_literals because of a bug in py2 setuptools # where package_data is expected to be str and not unicode. from __future__ import absolute_import, division, print_function import sys try: from ez_setup import use_setuptools use_setuptools() except ImportError: # Try to use ez_setup, but if not, continue anyway. The import is known # to fail when installing from a tar.gz. print('Could not import ez_setup', file=sys.stderr) from setuptools import setup # WARNING: This imposes limitations on requirements.txt such that the # full Pip syntax is not supported. See also # . install_reqs = [] with open('requirements.txt') as f: # pylint: disable=W1514 install_reqs += f.read().splitlines() setup_requires = [ # Pin pytest-runner to 5.2.0, since 5.3.0 uses `find_namespaces` directive, not supported in # Python 2.7 'pytest-runner == 5.2.0', ] # WARNING: This imposes limitations on test/requirements.txt such that the # full Pip syntax is not supported. See also # . test_reqs = [] with open('test/requirements.txt') as f: # pylint: disable=W1514 test_reqs += f.read().splitlines() with open('README.rst') as f: # pylint: disable=W1514 README = f.read() dist = setup( name='stone', version='3.3.1', install_requires=install_reqs, setup_requires=setup_requires, tests_require=test_reqs, entry_points={ 'console_scripts': ['stone=stone.cli:main'], }, packages=[ 'stone', 'stone.backends', 'stone.backends.python_rsrc', 'stone.frontend', 'stone.ir', ], package_data={ 'stone': ['py.typed'], }, zip_safe=False, author_email='kelkabany@dropbox.com', author='Ken Elkabany', description='Stone is an interface description language (IDL) for APIs.', license='MIT License', long_description=README, long_description_content_type='text/x-rst', maintainer_email='api-platform@dropbox.com', maintainer='Dropbox', url='https://github.com/dropbox/stone', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', 'Programming Language :: Python :: 3.7', 'Topic :: Software Development :: Code Generators', 'Topic :: Software Development :: Libraries :: Python Modules', ], ) stone-3.3.1/stone/000077500000000000000000000000001417406541500140055ustar00rootroot00000000000000stone-3.3.1/stone/__init__.py000066400000000000000000000000001417406541500161040ustar00rootroot00000000000000stone-3.3.1/stone/backend.py000066400000000000000000000457741417406541500157670ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from abc import ABCMeta, abstractmethod from contextlib import contextmanager import argparse import logging import os import six import textwrap from stone.frontend.ir_generator import doc_ref_re from stone.ir import ( is_alias, resolve_aliases, strip_alias ) _MYPY = False if _MYPY: from stone.ir import Api import typing # pylint: disable=import-error,useless-suppression # Generic Dict key-val types DelimTuple = typing.Tuple[typing.Text, typing.Text] K = typing.TypeVar('K') V = typing.TypeVar('V') def remove_aliases_from_api(api): # Resolve nested aliases from each namespace first. This way, when we replace an alias with # its source later on, it too is alias free. for namespace in api.namespaces.values(): for alias in namespace.aliases: # This loops through each alias type chain, resolving each (nested) alias # to its underlying type at the end of the chain (see resolve_aliases fn). # # It will continue until it no longer encounters a type # with a data_type attribute - this ensures it resolves aliases # that are subtypes of composites e.g. Lists curr_type = alias while hasattr(curr_type, 'data_type'): curr_type.data_type = resolve_aliases(curr_type.data_type) curr_type = curr_type.data_type # Remove alias layers from each data type for namespace in api.namespaces.values(): for data_type in namespace.data_types: for field in data_type.fields: strip_alias(field) for route in namespace.routes: # Strip inner aliases strip_alias(route.arg_data_type) strip_alias(route.result_data_type) strip_alias(route.error_data_type) # Strip top-level aliases if is_alias(route.arg_data_type): route.arg_data_type = route.arg_data_type.data_type if is_alias(route.result_data_type): route.result_data_type = route.result_data_type.data_type if is_alias(route.error_data_type): route.error_data_type = route.error_data_type.data_type # Clear aliases namespace.aliases = [] namespace.alias_by_name = {} return api @six.add_metaclass(ABCMeta) class Backend(object): """ The parent class for all backends. All backends should extend this class to be recognized as such. You will want to implement the generate() function to do the generation that you need. Here's roughly what you need to do in generate(). 1. Use the context manager output_to_relative_path() to specify an output file. with output_to_relative_path('generated_code.py'): ... 2. Use the family of emit*() functions to write to the output file. The target_folder_path attribute is the path to the folder where all generated files should be created. """ # Can be overridden by a subclass tabs_for_indents = False # Can be overridden with an argparse.ArgumentParser object. cmdline_parser = None # type: argparse.ArgumentParser # Can be overridden by a subclass. If true, stone.data_type.Alias # objects will be present in the API object. If false, aliases are masked # by replacing them with duplicate type definitions as the source type. # For backwards compatibility with existing backends defaults to false. preserve_aliases = False def __init__(self, target_folder_path, args): # type: (str, typing.Optional[typing.Sequence[str]]) -> None """ Args: target_folder_path (str): Path to the folder where all generated files should be created. """ self.logger = logging.getLogger('Backend<%s>' % self.__class__.__name__) self.target_folder_path = target_folder_path # Output is a list of strings that should be concatenated together for # the final output. self.output = [] # type: typing.List[typing.Text] self.lineno = 1 self.cur_indent = 0 self.positional_placeholders = [] # type: typing.List[typing.Text] self.named_placeholders = {} # type: typing.Dict[typing.Text, typing.Text] self.args = None # type: typing.Optional[argparse.Namespace] if self.cmdline_parser: assert isinstance(self.cmdline_parser, argparse.ArgumentParser), ( 'expected cmdline_parser to be ArgumentParser, got %r' % self.cmdline_parser) try: self.args = self.cmdline_parser.parse_args(args) except SystemExit: print('Note: This is for backend-specific arguments which ' 'follow arguments to Stone after a "--" delimiter.') raise @abstractmethod def generate(self, api): # type: (Api) -> None """ Subclasses should override this method. It's the entry point that is invoked by the rest of the toolchain. Args: api (stone.api.Api): The API specification. """ raise NotImplementedError @contextmanager def output_to_relative_path(self, relative_path, mode='wb'): # type: (typing.Text, typing.Text) -> typing.Iterator[None] """ Sets up backend so that all emits are directed towards the new file created at :param:`relative_path`. Clears the output buffer on enter and exit. """ full_path = os.path.join(self.target_folder_path, relative_path) directory = os.path.dirname(full_path) if not os.path.exists(directory): self.logger.info('Creating %s', directory) os.makedirs(directory) self.logger.info('Generating %s', full_path) self.clear_output_buffer() yield with open(full_path, mode) as f: # pylint: disable=unspecified-encoding f.write(self.output_buffer_to_string().encode('utf-8')) self.clear_output_buffer() def output_buffer_to_string(self): # type: () -> typing.Text """Returns the contents of the output buffer as a string.""" return ''.join(self.output).format( *self.positional_placeholders, **self.named_placeholders) def clear_output_buffer(self): self.output = [] self.positional_placeholders = [] self.named_placeholders = {} def indent_step(self): # type: () -> int """ Returns the size of a single indentation step. """ return 1 if self.tabs_for_indents else 4 @contextmanager def indent(self, dent=None): # type: (typing.Optional[int]) -> typing.Iterator[None] """ For the duration of the context manager, indentation will be increased by dent. Dent is in units of spaces or tabs depending on the value of the class variable tabs_for_indents. If dent is None, indentation will increase by either four spaces or one tab. """ assert dent is None or dent >= 0, 'dent must be >= 0.' if dent is None: dent = self.indent_step() self.cur_indent += dent yield self.cur_indent -= dent def make_indent(self): # type: () -> typing.Text """ Returns a string representing the current indentation. Indents can be either spaces or tabs, depending on the value of the class variable tabs_for_indents. """ if self.tabs_for_indents: return '\t' * self.cur_indent else: return ' ' * self.cur_indent @contextmanager def capture_emitted_output(self, output_buffer): # type: (six.StringIO) -> typing.Iterator[None] original_output = self.output self.output = [] yield output_buffer.write(''.join(self.output)) self.output = original_output def emit_raw(self, s): # type: (typing.Text) -> None """ Adds the input string to the output buffer. The string must end in a newline. It may contain any number of newline characters. No indentation is generated. """ self.lineno += s.count('\n') self._append_output(s.replace('{', '{{').replace('}', '}}')) if len(s) > 0 and s[-1] != '\n': raise AssertionError( 'Input string to emit_raw must end with a newline.') def _append_output(self, s): # type: (typing.Text) -> None self.output.append(s) def emit(self, s=''): # type: (typing.Text) -> None """ Adds indentation, then the input string, and lastly a newline to the output buffer. If s is an empty string (default) then an empty line is created with no indentation. """ assert isinstance(s, six.text_type), 's must be a unicode string' assert '\n' not in s, \ 'String to emit cannot contain newline strings.' if s: self.emit_raw('%s%s\n' % (self.make_indent(), s)) else: self.emit_raw('\n') def emit_wrapped_text( self, s, # type: typing.Text prefix='', # type: typing.Text initial_prefix='', # type: typing.Text subsequent_prefix='', # type: typing.Text width=80, # type: int break_long_words=False, # type: bool break_on_hyphens=False # type: bool ): # type: (...) -> None """ Adds the input string to the output buffer with indentation and wrapping. The wrapping is performed by the :func:`textwrap.fill` Python library function. Args: s (str): The input string to wrap. prefix (str): The string to prepend to *every* line. initial_prefix (str): The string to prepend to the first line of the wrapped string. Note that the current indentation is already added to each line. subsequent_prefix (str): The string to prepend to every line after the first. Note that the current indentation is already added to each line. width (int): The target width of each line including indentation and text. break_long_words (bool): Break words longer than width. If false, those words will not be broken, and some lines might be longer than width. break_on_hyphens (bool): Allow breaking hyphenated words. If true, wrapping will occur preferably on whitespaces and right after hyphens part of compound words. """ indent = self.make_indent() prefix = indent + prefix self.emit_raw(textwrap.fill(s, initial_indent=prefix + initial_prefix, subsequent_indent=prefix + subsequent_prefix, width=width, break_long_words=break_long_words, break_on_hyphens=break_on_hyphens, ) + '\n') def emit_placeholder(self, s=''): # type: (typing.Text) -> None """ Emits replacements fields that can be used to format the output string later. """ self._append_output('{%s}' % s) def add_positional_placeholder(self, s): # type: (typing.Text) -> None """ Format replacement fields corresponding to empty calls to emit_placeholder. """ self.positional_placeholders.append(s) def add_named_placeholder(self, name, s): # type: (typing.Text, typing.Text) -> None """ Format replacement fields corresponding to non-empty calls to emit_placeholder. """ self.named_placeholders[name] = s @classmethod def process_doc(cls, doc, handler): # type: (str, typing.Callable[[str, str], str]) -> typing.Text """ Helper for parsing documentation references in Stone docstrings and replacing them with more suitable annotations for the generated output. Args: doc (str): A Stone docstring. handler: A function with the following signature: `(tag: str, value: str) -> str`. It will be called for every reference found in the docstring with the tag and value parsed for you. The returned string will be substituted in the docstring in place of the reference. """ assert isinstance(doc, six.text_type), \ 'Expected string (unicode in PY2), got %r.' % type(doc) cur_index = 0 parts = [] for match in doc_ref_re.finditer(doc): # Append the part of the doc that is not part of any reference. start, end = match.span() parts.append(doc[cur_index:start]) cur_index = end # Call the handler with the next tag and value. tag = match.group('tag') val = match.group('val') sub = handler(tag, val) parts.append(sub) parts.append(doc[cur_index:]) return ''.join(parts) class CodeBackend(Backend): """ Extend this instead of :class:`Backend` when generating source code. Contains helper functions specific to code generation. """ # pylint: disable=abstract-method def filter_out_none_valued_keys(self, d): # type: (typing.Dict[K, V]) -> typing.Dict[K, V] """Given a dict, returns a new dict with all the same key/values except for keys that had values of None.""" new_d = {} for k, v in d.items(): if v is not None: new_d[k] = v return new_d def generate_multiline_list( self, items, # type: typing.List[typing.Text] before='', # type: typing.Text after='', # type: typing.Text delim=('(', ')'), # type: DelimTuple compact=True, # type: bool sep=',', # type: typing.Text skip_last_sep=False # type: bool ): # type: (...) -> None """ Given a list of items, emits one item per line. This is convenient for function prototypes and invocations, as well as for instantiating arrays, sets, and maps in some languages. TODO(kelkabany): A backend that uses tabs cannot be used with this if compact is false. Args: items (list[str]): Should contain the items to generate a list of. before (str): The string to come before the list of items. after (str): The string to follow the list of items. delim (str, str): The first element is added immediately following `before`. The second element is added prior to `after`. compact (bool): In compact mode, the enclosing parentheses are on the same lines as the first and last list item. sep (str): The string that follows each list item when compact is true. If compact is false, the separator is omitted for the last item. skip_last_sep (bool): When compact is false, whether the last line should have a trailing separator. Ignored when compact is true. """ assert len(delim) == 2 and isinstance(delim[0], six.text_type) and \ isinstance(delim[1], six.text_type), 'delim must be a tuple of two unicode strings.' if len(items) == 0: self.emit(before + delim[0] + delim[1] + after) return if len(items) == 1: self.emit(before + delim[0] + items[0] + delim[1] + after) return if compact: self.emit(before + delim[0] + items[0] + sep) def emit_list(items): items = items[1:] for (i, item) in enumerate(items): if i == len(items) - 1: self.emit(item + delim[1] + after) else: self.emit(item + sep) if before or delim[0]: with self.indent(len(before) + len(delim[0])): emit_list(items) else: emit_list(items) else: if before or delim[0]: self.emit(before + delim[0]) with self.indent(): for (i, item) in enumerate(items): if i == len(items) - 1 and skip_last_sep: self.emit(item) else: self.emit(item + sep) if delim[1] or after: self.emit(delim[1] + after) elif delim[1]: self.emit(delim[1]) @contextmanager def block( self, before='', # type: typing.Text after='', # type: typing.Text delim=('{', '}'), # type: DelimTuple dent=None, # type: typing.Optional[int] allman=False # type: bool ): # type: (...) -> typing.Iterator[None] """ A context manager that emits configurable lines before and after an indented block of text. This is convenient for class and function definitions in some languages. Args: before (str): The string to be output in the first line which is not indented.. after (str): The string to be output in the last line which is not indented. delim (str, str): The first element is added immediately following `before` and a space. The second element is added prior to a space and then `after`. dent (int): The amount to indent the block. If none, the default indentation increment is used (four spaces or one tab). allman (bool): Indicates whether to use `Allman` style indentation, or the default `K&R` style. If there is no `before` string this is ignored. For more details about indent styles see http://en.wikipedia.org/wiki/Indent_style """ assert len(delim) == 2, 'delim must be a tuple of length 2' assert (isinstance(delim[0], (six.text_type, type(None))) and isinstance(delim[1], (six.text_type, type(None)))), ( 'delim must be a tuple of two optional strings.') if before and not allman: if delim[0] is not None: self.emit('{} {}'.format(before, delim[0])) else: self.emit(before) else: if before: self.emit(before) if delim[0] is not None: self.emit(delim[0]) with self.indent(dent): yield if delim[1] is not None: self.emit(delim[1] + after) else: self.emit(after) stone-3.3.1/stone/backends/000077500000000000000000000000001417406541500155575ustar00rootroot00000000000000stone-3.3.1/stone/backends/__init__.py000066400000000000000000000000001417406541500176560ustar00rootroot00000000000000stone-3.3.1/stone/backends/helpers.py000066400000000000000000000032201417406541500175700ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import re _split_words_capitalization_re = re.compile( '^[a-z0-9]+|[A-Z][a-z0-9]+|[A-Z]+(?=[A-Z][a-z0-9])|[A-Z]+$' ) _split_words_dashes_re = re.compile('[-_/]+') def split_words(name): """ Splits name based on capitalization, dashes, and underscores. Example: 'GetFile' -> ['Get', 'File'] Example: 'get_file' -> ['get', 'file'] """ all_words = [] for word in re.split(_split_words_dashes_re, name): vals = _split_words_capitalization_re.findall(word) if vals: all_words.extend(vals) else: all_words.append(word) return all_words def fmt_camel(name): """ Converts name to lower camel case. Words are identified by capitalization, dashes, and underscores. """ words = split_words(name) assert len(words) > 0 first = words.pop(0).lower() return first + ''.join([word.capitalize() for word in words]) def fmt_dashes(name): """ Converts name to words separated by dashes. Words are identified by capitalization, dashes, and underscores. """ return '-'.join([word.lower() for word in split_words(name)]) def fmt_pascal(name): """ Converts name to pascal case. Words are identified by capitalization, dashes, and underscores. """ return ''.join([word.capitalize() for word in split_words(name)]) def fmt_underscores(name): """ Converts name to words separated by underscores. Words are identified by capitalization, dashes, and underscores. """ return '_'.join([word.lower() for word in split_words(name)]) stone-3.3.1/stone/backends/js_client.py000066400000000000000000000132271417406541500201100ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression from stone.ir import ApiNamespace # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any from stone.backend import CodeBackend from stone.backends.js_helpers import ( check_route_name_conflict, fmt_error_type, fmt_func, fmt_obj, fmt_type, fmt_url, ) from stone.ir import Void _cmdline_parser = argparse.ArgumentParser(prog='js-client-backend') _cmdline_parser.add_argument( 'filename', help=('The name to give the single Javascript file that is created and ' 'contains all of the routes.'), ) _cmdline_parser.add_argument( '-c', '--class-name', type=str, help=('The name of the class the generated functions will be attached to. ' 'The name will be added to each function documentation, which makes ' 'it available for tools like JSDoc.'), ) _cmdline_parser.add_argument( '--wrap-response-in', type=str, default='', help=('Wraps the response in a response class') ) _cmdline_parser.add_argument( '--wrap-error-in', type=str, default='', help=('Wraps the error in an error class') ) _header = """\ // Auto-generated by Stone, do not modify. var routes = {}; """ class JavascriptClientBackend(CodeBackend): """Generates a single Javascript file with all of the routes defined.""" cmdline_parser = _cmdline_parser # Instance var of the current namespace being generated cur_namespace = None # type: typing.Optional[ApiNamespace] preserve_aliases = True def generate(self, api): # first check for route name conflict with self.output_to_relative_path(self.args.filename): self.emit_raw(_header) for namespace in api.namespaces.values(): # Hack: needed for _docf() self.cur_namespace = namespace check_route_name_conflict(namespace) for route in namespace.routes: self._generate_route(api.route_schema, namespace, route) self.emit() self.emit('export { routes };') def _generate_route(self, route_schema, namespace, route): function_name = fmt_func(namespace.name + '_' + route.name, route.version) self.emit() self.emit('/**') if route.doc: self.emit_wrapped_text(self.process_doc(route.doc, self._docf), prefix=' * ') if self.args.class_name: self.emit(' * @function {}#{}'.format(self.args.class_name, function_name)) if route.deprecated: self.emit(' * @deprecated') return_type = None if self.args.wrap_response_in: return_type = '%s<%s>' % (self.args.wrap_response_in, fmt_type(route.result_data_type)) else: return_type = fmt_type(route.result_data_type) if route.arg_data_type.__class__ != Void: self.emit(' * @arg {%s} arg - The request parameters.' % fmt_type(route.arg_data_type)) self.emit(' * @returns {Promise.<%s, %s>}' % (return_type, fmt_error_type(route.error_data_type, self.args.wrap_error_in))) self.emit(' */') if route.arg_data_type.__class__ != Void: self.emit('routes.%s = function (arg) {' % (function_name)) else: self.emit('routes.%s = function () {' % (function_name)) with self.indent(dent=2): url = fmt_url(namespace.name, route.name, route.version) if route_schema.fields: additional_args = [] for field in route_schema.fields: additional_args.append(fmt_obj(route.attrs[field.name])) if route.arg_data_type.__class__ != Void: self.emit( "return this.request('{}', arg, {});".format( url, ', '.join(additional_args))) else: self.emit( "return this.request('{}', null, {});".format( url, ', '.join(additional_args))) else: if route.arg_data_type.__class__ != Void: self.emit( 'return this.request("%s", arg);' % url) else: self.emit( 'return this.request("%s", null);' % url) self.emit('};') def _docf(self, tag, val): """ Callback used as the handler argument to process_docs(). This converts Stone doc references to JSDoc-friendly annotations. """ # TODO(kelkabany): We're currently just dropping all doc ref tags ... # NOTE(praneshp): ... except for versioned routes if tag == 'route': if ':' in val: val, version = val.split(':', 1) version = int(version) else: version = 1 url = fmt_url(self.cur_namespace.name, val, version) # NOTE: In js, for comments, we drop the namespace name and the '/' when # documenting URLs return url[(len(self.cur_namespace.name) + 1):] return val stone-3.3.1/stone/backends/js_helpers.py000066400000000000000000000065041417406541500202740ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json import six from stone.ir import ( Boolean, Bytes, Float32, Float64, Int32, Int64, List, String, Timestamp, UInt32, UInt64, Void, is_list_type, is_struct_type, is_user_defined_type, ) from stone.backends.helpers import ( fmt_camel, fmt_pascal, ) _base_type_table = { Boolean: 'boolean', Bytes: 'string', Float32: 'number', Float64: 'number', Int32: 'number', Int64: 'number', List: 'Array', String: 'string', UInt32: 'number', UInt64: 'number', Timestamp: 'Timestamp', Void: 'void', } def fmt_obj(o): if isinstance(o, six.text_type): # Prioritize single-quoted strings per JS style guides. return repr(o).lstrip('u') else: return json.dumps(o, indent=2) def fmt_error_type(data_type, wrap_error_in=''): """ Converts the error type into a JSDoc type. """ return '%s.<%s>' % ( (wrap_error_in if (wrap_error_in != '') else 'Error'), fmt_type(data_type) ) def fmt_type_name(data_type): """ Returns the JSDoc name for the given data type. (Does not attempt to enumerate subtypes.) """ if is_user_defined_type(data_type): return fmt_pascal('%s%s' % (data_type.namespace.name, data_type.name)) else: fmted_type = _base_type_table.get(data_type.__class__, 'Object') if is_list_type(data_type): fmted_type += '.<' + fmt_type(data_type.data_type) + '>' return fmted_type def fmt_type(data_type): """ Returns a JSDoc annotation for a data type. May contain a union of enumerated subtypes. """ if is_struct_type(data_type) and data_type.has_enumerated_subtypes(): possible_types = [] possible_subtypes = data_type.get_all_subtypes_with_tags() for _, subtype in possible_subtypes: possible_types.append(fmt_type_name(subtype)) if data_type.is_catch_all(): possible_types.append(fmt_type_name(data_type)) return fmt_jsdoc_union(possible_types) else: return fmt_type_name(data_type) def fmt_jsdoc_union(type_strings): """ Returns a JSDoc union of the given type strings. """ return '(' + '|'.join(type_strings) + ')' if len(type_strings) > 1 else type_strings[0] def fmt_func(name, version): if version == 1: return fmt_camel(name) return fmt_camel(name) + 'V{}'.format(version) def fmt_url(namespace_name, route_name, route_version): if route_version != 1: return '{}/{}_v{}'.format(namespace_name, route_name, route_version) else: return '{}/{}'.format(namespace_name, route_name) def fmt_var(name): return fmt_camel(name) def check_route_name_conflict(namespace): """ Check name conflicts among generated route definitions. Raise a runtime exception when a conflict is encountered. """ route_by_name = {} for route in namespace.routes: route_name = fmt_func(route.name, version=route.version) if route_name in route_by_name: other_route = route_by_name[route_name] raise RuntimeError( 'There is a name conflict between {!r} and {!r}'.format(other_route, route)) route_by_name[route_name] = route stone-3.3.1/stone/backends/js_types.py000066400000000000000000000263051417406541500177770ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json import six import sys from stone.ir import ( is_user_defined_type, is_union_type, is_struct_type, is_void_type, unwrap, ) from stone.backend import CodeBackend from stone.backends.js_helpers import ( fmt_jsdoc_union, fmt_type, fmt_type_name, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any _cmdline_parser = argparse.ArgumentParser(prog='js-types-backend') _cmdline_parser.add_argument( 'filename', help=('The name to give the single Javascript file that is created and ' 'contains all of the JSDoc types.'), ) _cmdline_parser.add_argument( '-e', '--extra-arg', action='append', type=str, default=[], help=("Additional properties to add to a route's argument type based " "on if the route has a certain attribute set. Format (JSON): " '{"match": ["ROUTE_ATTR", ROUTE_VALUE_TO_MATCH], ' '"arg_name": "ARG_NAME", "arg_type": "ARG_TYPE", ' '"arg_docstring": "ARG_DOCSTRING"}'), ) _header = """\ // Auto-generated by Stone, do not modify. /** * An Error object returned from a route. * @typedef {Object} Error * @property {string} error_summary - Text summary of the error. * @property {T} error - The error object. * @property {UserMessage} user_message - An optional field. If present, it includes a message that can be shown directly to the end user of your app. You should show this message if your app is unprepared to programmatically handle the error returned by an endpoint. * @template T */ /** * User-friendly error message. * @typedef {Object} UserMessage * @property {string} text - The message. * @property {string} locale */ /** * @typedef {string} Timestamp */ """ class JavascriptTypesBackend(CodeBackend): """Generates a single Javascript file with all of the data types defined in JSDoc.""" cmdline_parser = _cmdline_parser preserve_aliases = True def generate(self, api): with self.output_to_relative_path(self.args.filename): self.emit_raw(_header) extra_args = self._parse_extra_args(api, self.args.extra_arg) for namespace in api.namespaces.values(): for data_type in namespace.data_types: self._generate_type(data_type, extra_args.get(data_type, [])) def _parse_extra_args(self, api, extra_args_raw): """ Parses extra arguments into a map keyed on particular data types. """ extra_args = {} def die(m, extra_arg_raw): print('Invalid --extra-arg:%s: %s' % (m, extra_arg_raw), file=sys.stderr) sys.exit(1) for extra_arg_raw in extra_args_raw: try: extra_arg = json.loads(extra_arg_raw) except ValueError as e: die(str(e), extra_arg_raw) # Validate extra_arg JSON blob if 'match' not in extra_arg: die('No match key', extra_arg_raw) elif (not isinstance(extra_arg['match'], list) or len(extra_arg['match']) != 2): die('match key is not a list of two strings', extra_arg_raw) elif (not isinstance(extra_arg['match'][0], six.text_type) or not isinstance(extra_arg['match'][1], six.text_type)): print(type(extra_arg['match'][0])) die('match values are not strings', extra_arg_raw) elif 'arg_name' not in extra_arg: die('No arg_name key', extra_arg_raw) elif not isinstance(extra_arg['arg_name'], six.text_type): die('arg_name is not a string', extra_arg_raw) elif 'arg_type' not in extra_arg: die('No arg_type key', extra_arg_raw) elif not isinstance(extra_arg['arg_type'], six.text_type): die('arg_type is not a string', extra_arg_raw) elif ('arg_docstring' in extra_arg and not isinstance(extra_arg['arg_docstring'], six.text_type)): die('arg_docstring is not a string', extra_arg_raw) attr_key, attr_val = extra_arg['match'][0], extra_arg['match'][1] extra_args.setdefault(attr_key, {})[attr_val] = \ (extra_arg['arg_name'], extra_arg['arg_type'], extra_arg.get('arg_docstring')) # Extra arguments, keyed on data type objects. extra_args_for_types = {} # Locate data types that contain extra arguments for namespace in api.namespaces.values(): for route in namespace.routes: extra_parameters = [] if is_user_defined_type(route.arg_data_type): for attr_key in route.attrs: if attr_key not in extra_args: continue attr_val = route.attrs[attr_key] if attr_val in extra_args[attr_key]: extra_parameters.append(extra_args[attr_key][attr_val]) if len(extra_parameters) > 0: extra_args_for_types[route.arg_data_type] = extra_parameters return extra_args_for_types def _generate_type(self, data_type, extra_parameters): if is_struct_type(data_type): self._generate_struct(data_type, extra_parameters) elif is_union_type(data_type): self._generate_union(data_type) def _emit_jsdoc_header(self, doc=None): self.emit() self.emit('/**') if doc: self.emit_wrapped_text(self.process_doc(doc, self._docf), prefix=' * ') def _generate_struct(self, struct_type, extra_parameters=None, nameOverride=None): """ Emits a JSDoc @typedef for a struct. """ extra_parameters = extra_parameters if extra_parameters is not None else [] self._emit_jsdoc_header(struct_type.doc) self.emit( ' * @typedef {Object} %s' % ( nameOverride if nameOverride else fmt_type_name(struct_type) ) ) # Some structs can explicitly list their subtypes. These structs # have a .tag field that indicate which subtype they are. if struct_type.is_member_of_enumerated_subtypes_tree(): if struct_type.has_enumerated_subtypes(): # This struct is the parent to multiple subtypes. # Determine all of the possible values of the .tag # property. tag_values = [] for tags, _ in struct_type.get_all_subtypes_with_tags(): for tag in tags: tag_values.append('"%s"' % tag) jsdoc_tag_union = fmt_jsdoc_union(tag_values) txt = '@property {%s} .tag - Tag identifying the subtype variant.' % \ jsdoc_tag_union self.emit_wrapped_text(txt) else: # This struct is a particular subtype. Find the applicable # .tag value from the parent type, which may be an # arbitrary number of steps up the inheritance hierarchy. parent = struct_type.parent_type while not parent.has_enumerated_subtypes(): parent = parent.parent_type # parent now contains the closest parent type in the # inheritance hierarchy that has enumerated subtypes. # Determine which subtype this is. for subtype in parent.get_enumerated_subtypes(): if subtype.data_type == struct_type: txt = '@property {\'%s\'} [.tag] - Tag identifying ' \ 'this subtype variant. This field is only ' \ 'present when needed to discriminate ' \ 'between multiple possible subtypes.' % \ subtype.name self.emit_wrapped_text(txt) break for param_name, param_type, param_docstring in extra_parameters: param_docstring = ' - %s' % param_docstring if param_docstring else '' self.emit_wrapped_text( '@property {%s} [%s]%s' % ( param_type, param_name, param_docstring, ), prefix=' * ', ) # NOTE: JSDoc @typedef does not support inheritance. Using @class would be inappropriate, # since these are not nominal types backed by a constructor. Thus, we emit all_fields, # which includes fields on parent types. for field in struct_type.all_fields: field_doc = ' - ' + field.doc if field.doc else '' field_type, nullable, _ = unwrap(field.data_type) field_js_type = fmt_type(field_type) # Translate nullable types into optional properties. field_name = '[' + field.name + ']' if nullable else field.name self.emit_wrapped_text( '@property {%s} %s%s' % ( field_js_type, field_name, self.process_doc(field_doc, self._docf), ), prefix=' * ', ) self.emit(' */') def _generate_union(self, union_type): """ Emits a JSDoc @typedef for a union type. """ union_name = fmt_type_name(union_type) self._emit_jsdoc_header(union_type.doc) self.emit(' * @typedef {Object} %s' % union_name) variant_types = [] for variant in union_type.all_fields: variant_types.append("'%s'" % variant.name) variant_data_type, _, _ = unwrap(variant.data_type) # Don't emit fields for void types. if not is_void_type(variant_data_type): variant_doc = ' - Available if .tag is %s.' % variant.name if variant.doc: variant_doc += ' ' + variant.doc self.emit_wrapped_text( '@property {%s} [%s]%s' % ( fmt_type(variant_data_type), variant.name, variant_doc, ), prefix=' * ', ) jsdoc_tag_union = fmt_jsdoc_union(variant_types) self.emit(' * @property {%s} .tag - Tag identifying the union variant.' % jsdoc_tag_union) self.emit(' */') def _docf(self, tag, val): # pylint: disable=unused-argument """ Callback used as the handler argument to process_docs(). This converts Stone doc references to JSDoc-friendly annotations. """ # TODO(kelkabany): We're currently just dropping all doc ref tags. return val stone-3.3.1/stone/backends/obj_c.py000066400000000000000000000233421417406541500172110ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from contextlib import contextmanager from stone.ir import ( is_list_type, is_map_type, is_struct_type, is_union_type, is_nullable_type, is_user_defined_type, is_void_type, unwrap_nullable, ) from stone.backend import CodeBackend from stone.backends.obj_c_helpers import ( fmt_camel_upper, fmt_class, fmt_class_prefix, fmt_import, ) stone_warning = """\ /// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// /// Auto-generated by Stone, do not modify. /// """ # This will be at the top of the generated file. base_file_comment = """\ {}\ """.format(stone_warning) undocumented = '(no description).' comment_prefix = '/// ' class ObjCBaseBackend(CodeBackend): """Wrapper class over Stone generator for Obj C logic.""" # pylint: disable=abstract-method @contextmanager def block_m(self, class_name): with self.block( '@implementation {}'.format(class_name), delim=('', '@end'), dent=0): self.emit() yield @contextmanager def block_h_from_data_type(self, data_type, protocol=None): assert is_user_defined_type(data_type), \ 'Expected user-defined type, got %r' % type(data_type) if not protocol: extensions = [] if data_type.parent_type and is_struct_type(data_type): extensions.append(fmt_class_prefix(data_type.parent_type)) else: if is_union_type(data_type): # Use a handwritten base class extensions.append('NSObject') else: extensions.append('NSObject') extend_suffix = ' : {}'.format( ', '.join(extensions)) if extensions else '' else: base = fmt_class_prefix(data_type.parent_type) if ( data_type.parent_type and not is_union_type(data_type)) else 'NSObject' extend_suffix = ' : {} <{}>'.format(base, ', '.join(protocol)) with self.block( '@interface {}{}'.format( fmt_class_prefix(data_type), extend_suffix), delim=('', '@end'), dent=0): self.emit() yield @contextmanager def block_h(self, class_name, protocol=None, extensions=None, protected=None): if not extensions: extensions = ['NSObject'] if not protocol: extend_suffix = ' : {}'.format(', '.join(extensions)) else: extend_suffix = ' : {} <{}>'.format(', '.join(extensions), fmt_class(protocol)) base_interface_str = '@interface {}{} {{' if protected else '@interface {}{}' with self.block( base_interface_str.format(class_name, extend_suffix), delim=('', '@end'), dent=0): if protected: with self.block('', delim=('', '')): self.emit('@protected') for field_name, field_type in protected: self.emit('{} _{};'.format(field_type, field_name)) self.emit('}') self.emit() yield @contextmanager def block_init(self): with self.block('if (self)'): yield self.emit('return self;') @contextmanager def block_func(self, func, args=None, return_type='void', class_func=False): args = args if args is not None else [] modifier = '-' if not class_func else '+' base_string = '{} ({}){}:{}' if args else '{} ({}){}' signature = base_string.format(modifier, return_type, func, args) with self.block(signature): yield def _get_imports_m(self, data_types, default_imports): """Emits all necessary implementation file imports for the given Stone data type.""" if not isinstance(data_types, list): data_types = [data_types] import_classes = default_imports for data_type in data_types: import_classes.append(fmt_class_prefix(data_type)) if data_type.parent_type: import_classes.append(fmt_class_prefix(data_type.parent_type)) if is_struct_type( data_type) and data_type.has_enumerated_subtypes(): for _, subtype in data_type.get_all_subtypes_with_tags(): import_classes.append(fmt_class_prefix(subtype)) for field in data_type.all_fields: data_type, _ = unwrap_nullable(field.data_type) # unpack list or map while is_list_type(data_type) or is_map_type(data_type): data_type = (data_type.value_data_type if is_map_type(data_type) else data_type.data_type) if is_user_defined_type(data_type): import_classes.append(fmt_class_prefix(data_type)) if import_classes: import_classes = list(set(import_classes)) import_classes.sort() return import_classes def _get_imports_h(self, data_types): """Emits all necessary header file imports for the given Stone data type.""" if not isinstance(data_types, list): data_types = [data_types] import_classes = [] for data_type in data_types: if is_user_defined_type(data_type): import_classes.append(fmt_class_prefix(data_type)) for field in data_type.all_fields: data_type, _ = unwrap_nullable(field.data_type) # unpack list or map while is_list_type(data_type) or is_map_type(data_type): data_type = (data_type.value_data_type if is_map_type(data_type) else data_type.data_type) if is_user_defined_type(data_type): import_classes.append(fmt_class_prefix(data_type)) import_classes = list(set(import_classes)) import_classes.sort() return import_classes def _generate_imports_h(self, import_classes): import_classes = list(set(import_classes)) import_classes.sort() for import_class in import_classes: self.emit('@class {};'.format(import_class)) if import_classes: self.emit() def _generate_imports_m(self, import_classes): import_classes = list(set(import_classes)) import_classes.sort() for import_class in import_classes: self.emit(fmt_import(import_class)) self.emit() def _generate_init_imports_h(self, data_type): self.emit('#import ') self.emit() self.emit('#import "DBSerializableProtocol.h"') if data_type.parent_type and not is_union_type(data_type): self.emit(fmt_import(fmt_class_prefix(data_type.parent_type))) self.emit() def _get_namespace_route_imports(self, namespace, include_route_args=True, include_route_deep_args=False): result = [] def _unpack_and_store_data_type(data_type): data_type, _ = unwrap_nullable(data_type) if is_list_type(data_type): while is_list_type(data_type): data_type, _ = unwrap_nullable(data_type.data_type) if not is_void_type(data_type) and is_user_defined_type(data_type): result.append(data_type) for route in namespace.routes: if include_route_args: data_type, _ = unwrap_nullable(route.arg_data_type) _unpack_and_store_data_type(data_type) elif include_route_deep_args: data_type, _ = unwrap_nullable(route.arg_data_type) if is_union_type(data_type) or is_list_type(data_type): _unpack_and_store_data_type(data_type) elif not is_void_type(data_type): for field in data_type.all_fields: data_type, _ = unwrap_nullable(field.data_type) if (is_struct_type(data_type) or is_union_type(data_type) or is_list_type(data_type)): _unpack_and_store_data_type(data_type) _unpack_and_store_data_type(route.result_data_type) _unpack_and_store_data_type(route.error_data_type) return result def _cstor_name_from_fields(self, fields): """Returns an Obj C appropriate name for a constructor based on the name of the first argument.""" if fields: return self._cstor_name_from_field(fields[0]) else: return 'initDefault' def _cstor_name_from_field(self, field): """Returns an Obj C appropriate name for a constructor based on the name of the supplied argument.""" return 'initWith{}'.format(fmt_camel_upper(field.name)) def _cstor_name_from_fields_names(self, fields_names): """Returns an Obj C appropriate name for a constructor based on the name of the first argument.""" if fields_names: return 'initWith{}'.format(fmt_camel_upper(fields_names[0][0])) else: return 'initDefault' def _struct_has_defaults(self, struct): """Returns whether the given struct has any default values.""" return [ f for f in struct.all_fields if f.has_default or is_nullable_type(f.data_type) ] stone-3.3.1/stone/backends/obj_c_client.py000066400000000000000000000626151417406541500205550ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json from stone.ir import ( is_nullable_type, is_struct_type, is_union_type, is_void_type, unwrap_nullable, ) from stone.backends.obj_c_helpers import ( fmt_alloc_call, fmt_camel_upper, fmt_class, fmt_class_prefix, fmt_func, fmt_func_args, fmt_func_args_declaration, fmt_func_call, fmt_import, fmt_property_str, fmt_route_obj_class, fmt_route_func, fmt_route_var, fmt_routes_class, fmt_signature, fmt_type, fmt_var, ) from stone.backends.obj_c import ( base_file_comment, comment_prefix, ObjCBaseBackend, stone_warning, undocumented, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any _cmdline_parser = argparse.ArgumentParser( prog='objc-client-backend', description=( 'Generates a ObjC class with an object for each namespace, and in each ' 'namespace object, a method for each route. This class assumes that the ' 'obj_c_types backend was used with the same output directory.'), ) _cmdline_parser.add_argument( '-m', '--module-name', required=True, type=str, help=( 'The name of the ObjC module to generate. Please exclude the {.h,.m} ' 'file extension.'), ) _cmdline_parser.add_argument( '-c', '--class-name', required=True, type=str, help=( 'The name of the ObjC class that contains an object for each namespace, ' 'and in each namespace object, a method for each route.')) _cmdline_parser.add_argument( '-t', '--transport-client-name', required=True, type=str, help='The name of the ObjC class that manages network API calls.', ) _cmdline_parser.add_argument( '-w', '--auth-type', type=str, help='The auth type of the client to generate.', ) _cmdline_parser.add_argument( '-y', '--client-args', required=True, type=str, help='The client-side route arguments to append to each route by style type.', ) _cmdline_parser.add_argument( '-z', '--style-to-request', required=True, type=str, help='The dict that maps a style type to a ObjC request object name.', ) class ObjCBackend(ObjCBaseBackend): """Generates ObjC client base that implements route interfaces.""" cmdline_parser = _cmdline_parser obj_name_to_namespace = {} # type: typing.Dict[str, int] namespace_to_has_routes = {} # type: typing.Dict[typing.Any, bool] def generate(self, api): for namespace in api.namespaces.values(): self.namespace_to_has_routes[namespace] = False if namespace.routes: for route in namespace.routes: if self._should_generate_route(route): self.namespace_to_has_routes[namespace] = True break for namespace in api.namespaces.values(): for data_type in namespace.linearize_data_types(): self.obj_name_to_namespace[data_type.name] = fmt_class_prefix( data_type) for namespace in api.namespaces.values(): if namespace.routes and self.namespace_to_has_routes[namespace]: import_classes = [ fmt_routes_class(namespace.name, self.args.auth_type), fmt_route_obj_class(namespace.name), '{}Protocol'.format(self.args.transport_client_name), 'DBStoneBase', 'DBRequestErrors', ] with self.output_to_relative_path('Routes/{}.m'.format( fmt_routes_class(namespace.name, self.args.auth_type))): self.emit_raw(stone_warning) imports_classes_m = import_classes + \ self._get_imports_m( self._get_namespace_route_imports(namespace), []) self._generate_imports_m(imports_classes_m) self._generate_routes_m(namespace) with self.output_to_relative_path('Routes/{}.h'.format( fmt_routes_class(namespace.name, self.args.auth_type))): self.emit_raw(base_file_comment) self.emit('#import ') self.emit() self.emit(fmt_import('DBTasks')) self.emit() import_classes_h = [ 'DBNilObject', ] import_classes_h = (import_classes_h + self._get_imports_h( self._get_namespace_route_imports( namespace, include_route_args=False, include_route_deep_args=True))) self._generate_imports_h(import_classes_h) self.emit( '@protocol {};'.format( self.args.transport_client_name), ) self.emit() self._generate_routes_h(namespace) with self.output_to_relative_path( 'Client/{}.m'.format(self.args.module_name)): self._generate_client_m(api) with self.output_to_relative_path( 'Client/{}.h'.format(self.args.module_name)): self._generate_client_h(api) def _generate_client_m(self, api): """Generates client base implementation file. For each namespace, the client will have an object field that encapsulates each route in the particular namespace.""" self.emit_raw(base_file_comment) import_classes = [self.args.module_name] import_classes += [ fmt_routes_class(ns.name, self.args.auth_type) for ns in api.namespaces.values() if ns.routes and self.namespace_to_has_routes[ns] ] import_classes.append( '{}Protocol'.format(self.args.transport_client_name)) self._generate_imports_m(import_classes) with self.block_m(self.args.class_name): client_args = fmt_func_args_declaration( [('client', 'id<{}>'.format(self.args.transport_client_name))]) with self.block_func( func='initWithTransportClient', args=client_args, return_type='instancetype'): self.emit('self = [super init];') with self.block_init(): self.emit('_transportClient = client;') for namespace in api.namespaces.values(): if namespace.routes and self.namespace_to_has_routes[namespace]: base_string = '_{}Routes = [[{} alloc] init:client];' self.emit( base_string.format( fmt_var(namespace.name), fmt_routes_class(namespace.name, self.args.auth_type))) def _generate_client_h(self, api): """Generates client base header file. For each namespace, the client will have an object field that encapsulates each route in the particular namespace.""" self.emit_raw(stone_warning) self.emit('#import ') import_classes = [ fmt_routes_class(ns.name, self.args.auth_type) for ns in api.namespaces.values() if ns.routes and self.namespace_to_has_routes[ns] ] import_classes.append('DBRequestErrors') import_classes.append('DBTasks') self._generate_imports_m(import_classes) self.emit() self.emit('NS_ASSUME_NONNULL_BEGIN') self.emit() self.emit('@protocol {};'.format(self.args.transport_client_name)) self.emit() self.emit(comment_prefix) description_str = ( 'Base client object that contains an instance field for ' 'each namespace, each of which contains references to all routes within ' 'that namespace. Fully-implemented API clients will inherit this class.' ) self.emit_wrapped_text(description_str, prefix=comment_prefix) self.emit(comment_prefix) with self.block_h( self.args.class_name, protected=[ ('transportClient', 'id<{}>'.format(self.args.transport_client_name)) ]): self.emit() for namespace in api.namespaces.values(): if namespace.routes and self.namespace_to_has_routes[namespace]: class_doc = 'Routes within the `{}` namespace.'.format( fmt_var(namespace.name)) self.emit_wrapped_text(class_doc, prefix=comment_prefix) prop = '{}Routes'.format(fmt_var(namespace.name)) typ = '{} *'.format( fmt_routes_class(namespace.name, self.args.auth_type)) self.emit(fmt_property_str(prop=prop, typ=typ)) self.emit() client_args = fmt_func_args_declaration( [('client', 'id<{}>'.format(self.args.transport_client_name))]) description_str = ( 'Initializes the `{}` object with a networking client.') self.emit_wrapped_text( description_str.format(self.args.class_name), prefix=comment_prefix) init_signature = fmt_signature( func='initWithTransportClient', args=client_args, return_type='instancetype') self.emit('{};'.format(init_signature)) self.emit() self.emit() self.emit('NS_ASSUME_NONNULL_END') def _auth_type_in_route(self, route, desired_auth_type): for auth_type in route.attrs.get('auth').split(','): if auth_type.strip() == desired_auth_type: return True return False def _route_is_special_noauth_case(self, route): return self._auth_type_in_route(route, 'noauth') and self.args.auth_type == 'user' def _should_generate_route(self, route): return (self._auth_type_in_route(route, self.args.auth_type) or self._route_is_special_noauth_case(route)) def _generate_routes_m(self, namespace): """Generates implementation file for namespace object that has as methods all routes within the namespace.""" with self.block_m( fmt_routes_class(namespace.name, self.args.auth_type)): init_args = fmt_func_args_declaration([( 'client', 'id<{}>'.format(self.args.transport_client_name))]) with self.block_func( func='init', args=init_args, return_type='instancetype'): self.emit('self = [super init];') with self.block_init(): self.emit('_client = client;') self.emit() style_to_request = json.loads(self.args.style_to_request) for route in namespace.routes: if not self._should_generate_route(route): continue route_type = route.attrs.get('style') client_args = json.loads(self.args.client_args) if route_type in client_args.keys(): for args_data in client_args[route_type]: task_type_key, type_data_dict = tuple(args_data) task_type_name = style_to_request[task_type_key] func_suffix = type_data_dict[0] extra_args = [ tuple(type_data[:-1]) for type_data in type_data_dict[1] ] if (is_struct_type(route.arg_data_type) and self._struct_has_defaults(route.arg_data_type)): route_args, _ = self._get_default_route_args( namespace, route) self._generate_route_m(route, namespace, route_args, extra_args, task_type_name, func_suffix) route_args, _ = self._get_route_args(namespace, route) self._generate_route_m(route, namespace, route_args, extra_args, task_type_name, func_suffix) else: task_type_name = style_to_request[route_type] if (is_struct_type(route.arg_data_type) and self._struct_has_defaults(route.arg_data_type)): route_args, _ = self._get_default_route_args( namespace, route) self._generate_route_m(route, namespace, route_args, [], task_type_name, '') route_args, _ = self._get_route_args(namespace, route) self._generate_route_m(route, namespace, route_args, [], task_type_name, '') def _generate_route_m(self, route, namespace, route_args, extra_args, task_type_name, func_suffix): """Generates route method implementation for the given route.""" user_args = list(route_args) transport_args = [ ('route', 'route'), ('arg', 'arg' if not is_void_type(route.arg_data_type) else 'nil'), ] for name, value, typ in extra_args: user_args.append((name, typ)) transport_args.append((name, value)) with self.block_func( func='{}{}'.format(fmt_route_func(route), func_suffix), args=fmt_func_args_declaration(user_args), return_type='{} *'.format(task_type_name)): self.emit('DBRoute *route = {}.{};'.format( fmt_route_obj_class(namespace.name), fmt_route_var(namespace.name, route))) if is_union_type(route.arg_data_type): self.emit('{} *arg = {};'.format( fmt_class_prefix(route.arg_data_type), fmt_var(route.arg_data_type.name))) elif not is_void_type(route.arg_data_type): init_call = fmt_func_call( caller=fmt_alloc_call( caller=fmt_class_prefix(route.arg_data_type)), callee=self._cstor_name_from_fields_names(route_args), args=fmt_func_args([(f[0], f[0]) for f in route_args])) self.emit('{} *arg = {};'.format( fmt_class_prefix(route.arg_data_type), init_call)) request_call = fmt_func_call( caller='self.client', callee='request{}'.format( fmt_camel_upper(route.attrs.get('style'))), args=fmt_func_args(transport_args)) self.emit('return {};'.format(request_call)) self.emit() def _generate_routes_h(self, namespace): """Generates header file for namespace object that has as methods all routes within the namespace.""" self.emit(comment_prefix) self.emit_wrapped_text( 'Routes for the `{}` namespace'.format(fmt_class(namespace.name)), prefix=comment_prefix) self.emit(comment_prefix) self.emit() self.emit('NS_ASSUME_NONNULL_BEGIN') self.emit() with self.block_h( fmt_routes_class(namespace.name, self.args.auth_type)): description_str = ( 'An instance of the networking client that each ' 'route will use to submit a request.') self.emit_wrapped_text(description_str, prefix=comment_prefix) self.emit( fmt_property_str( prop='client', typ='id<{}>'.format( self.args.transport_client_name))) self.emit() routes_obj_args = fmt_func_args_declaration( [('client', 'id<{}>'.format(self.args.transport_client_name))]) init_signature = fmt_signature( func='init', args=routes_obj_args, return_type='instancetype') description_str = ( 'Initializes the `{}` namespace container object ' 'with a networking client.') self.emit_wrapped_text( description_str.format( fmt_routes_class(namespace.name, self.args.auth_type)), prefix=comment_prefix) self.emit('{};'.format(init_signature)) self.emit() style_to_request = json.loads(self.args.style_to_request) for route in namespace.routes: if not self._should_generate_route(route): continue route_type = route.attrs.get('style') client_args = json.loads(self.args.client_args) if route_type in client_args.keys(): for args_data in client_args[route_type]: task_type_key, type_data_dict = tuple(args_data) task_type_name = style_to_request[task_type_key] func_suffix = type_data_dict[0] extra_args = [ tuple(type_data[:-1]) for type_data in type_data_dict[1] ] extra_docs = [(type_data[0], type_data[-1]) for type_data in type_data_dict[1]] if (is_struct_type(route.arg_data_type) and self._struct_has_defaults(route.arg_data_type)): route_args, doc_list = self._get_default_route_args( namespace, route, tag=True) self._generate_route_signature( route, namespace, route_args, extra_args, doc_list + extra_docs, task_type_name, func_suffix) route_args, doc_list = self._get_route_args( namespace, route, tag=True) self._generate_route_signature( route, namespace, route_args, extra_args, doc_list + extra_docs, task_type_name, func_suffix) else: task_type_name = style_to_request[route_type] if (is_struct_type(route.arg_data_type) and self._struct_has_defaults(route.arg_data_type)): route_args, doc_list = self._get_default_route_args( namespace, route, tag=True) self._generate_route_signature( route, namespace, route_args, [], doc_list, task_type_name, '') route_args, doc_list = self._get_route_args( namespace, route, tag=True) self._generate_route_signature(route, namespace, route_args, [], doc_list, task_type_name, '') self.emit() self.emit('NS_ASSUME_NONNULL_END') self.emit() def _generate_route_signature( self, route, namespace, # pylint: disable=unused-argument route_args, extra_args, doc_list, task_type_name, func_suffix): """Generates route method signature for the given route.""" for name, _, typ in extra_args: route_args.append((name, typ)) deprecated = 'DEPRECATED: ' if route.deprecated else '' func_name = '{}{}'.format(fmt_route_func(route), func_suffix) self.emit(comment_prefix) if route.doc: route_doc = self.process_doc(route.doc, self._docf) else: route_doc = 'The {} route'.format(func_name) self.emit_wrapped_text( deprecated + route_doc, prefix=comment_prefix, width=120) self.emit(comment_prefix) for name, doc in doc_list: self.emit_wrapped_text( '@param {} {}'.format(name, doc if doc else undocumented), prefix=comment_prefix, width=120) self.emit(comment_prefix) output = ( '@return Through the response callback, the caller will ' + 'receive a `{}` object on success or a `{}` object on failure.') output = output.format( fmt_type(route.result_data_type, tag=False, no_ptr=True), fmt_type(route.error_data_type, tag=False, no_ptr=True)) self.emit_wrapped_text(output, prefix=comment_prefix, width=120) self.emit(comment_prefix) result_type_str = fmt_type(route.result_data_type) if not is_void_type( route.result_data_type) else 'DBNilObject *' error_type_str = fmt_type(route.error_data_type) if not is_void_type( route.error_data_type) else 'DBNilObject *' return_type = '{}<{}, {}> *'.format(task_type_name, result_type_str, error_type_str) deprecated = self._get_deprecation_warning(route) route_signature = fmt_signature( func=func_name, args=fmt_func_args_declaration(route_args), return_type='{}'.format(return_type)) self.emit('{}{};'.format(route_signature, deprecated)) self.emit() def _get_deprecation_warning(self, route): """Returns a deprecation tag / message, if route is deprecated.""" result = '' if route.deprecated: msg = '{} is deprecated.'.format(fmt_route_func(route)) if route.deprecated.by: msg += ' Use {}.'.format(fmt_var(route.deprecated.by.name)) result = ' __deprecated_msg("{}")'.format(msg) return result def _get_route_args(self, namespace, route, tag=False): # pylint: disable=unused-argument """Returns a list of name / value string pairs representing the arguments for a particular route.""" data_type, _ = unwrap_nullable(route.arg_data_type) if is_struct_type(data_type): arg_list = [] for field in data_type.all_fields: arg_list.append((fmt_var(field.name), fmt_type( field.data_type, tag=tag, has_default=field.has_default))) doc_list = [(fmt_var(f.name), self.process_doc(f.doc, self._docf)) for f in data_type.fields if f.doc] elif is_union_type(data_type): arg_list = [(fmt_var(data_type.name), fmt_type( route.arg_data_type, tag=tag))] doc_list = [(fmt_var(data_type.name), self.process_doc(data_type.doc, self._docf) if data_type.doc else 'The {} union'.format( fmt_class(data_type .name)))] else: arg_list = [] doc_list = [] return arg_list, doc_list def _get_default_route_args( self, namespace, # pylint: disable=unused-argument route, tag=False): """Returns a list of name / value string pairs representing the default arguments for a particular route.""" data_type, _ = unwrap_nullable(route.arg_data_type) if is_struct_type(data_type): arg_list = [] for field in data_type.all_fields: if not field.has_default and not is_nullable_type( field.data_type): arg_list.append((fmt_var(field.name), fmt_type( field.data_type, tag=tag))) doc_list = ([(fmt_var(f.name), self.process_doc(f.doc, self._docf)) for f in data_type.fields if f.doc and not f.has_default and not is_nullable_type(f.data_type)]) else: arg_list = [] doc_list = [] return arg_list, doc_list def _docf(self, tag, val): if tag == 'route': return '`{}`'.format(fmt_func(val)) elif tag == 'field': if '.' in val: cls_name, field = val.split('.') return ('`{}` in `{}`'.format( fmt_var(field), self.obj_name_to_namespace[cls_name])) else: return fmt_var(val) elif tag in ('type', 'val', 'link'): return val else: return val stone-3.3.1/stone/backends/obj_c_helpers.py000066400000000000000000000307411417406541500207340ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import pprint from stone.ir import ( Boolean, Bytes, Float32, Float64, Int32, Int64, List, Map, String, Timestamp, UInt32, UInt64, Void, is_boolean_type, is_list_type, is_map_type, is_numeric_type, is_string_type, is_tag_ref, is_user_defined_type, is_void_type, unwrap_nullable, ) from .helpers import split_words # This file defines *stylistic* choices for Swift # (ie, that class names are UpperCamelCase and that variables are lowerCamelCase) _primitive_table = { Boolean: 'NSNumber *', Bytes: 'NSString *', Float32: 'NSNumber *', Float64: 'NSNumber *', Int32: 'NSNumber *', Int64: 'NSNumber *', List: 'NSArray', Map: 'NSDictionary', String: 'NSString *', Timestamp: 'NSDate *', UInt32: 'NSNumber *', UInt64: 'NSNumber *', Void: 'void', } _primitive_table_user_interface = { Boolean: 'BOOL', Bytes: 'NSString *', Float32: 'double', Float64: 'double', Int32: 'int', Int64: 'long', List: 'NSArray', Map: 'NSDictionary', String: 'NSString *', Timestamp: 'NSDate *', UInt32: 'unsigned int', UInt64: 'unsigned long', Void: 'void', } _serial_table = { Boolean: 'DBBoolSerializer', Bytes: 'DBStringSerializer', Float32: 'DBNSNumberSerializer', Float64: 'DBNSNumberSerializer', Int32: 'DBNSNumberSerializer', Int64: 'DBNSNumberSerializer', List: 'DBArraySerializer', Map: 'DBMapSerializer', String: 'DBStringSerializer', Timestamp: 'DBNSDateSerializer', UInt32: 'DBNSNumberSerializer', UInt64: 'DBNSNumberSerializer', } _validator_table = { Float32: 'numericValidator', Float64: 'numericValidator', Int32: 'numericValidator', Int64: 'numericValidator', List: 'arrayValidator', Map: 'mapValidator', String: 'stringValidator', UInt32: 'numericValidator', UInt64: 'numericValidator', } _wrapper_primitives = { Boolean, Float32, Float64, UInt32, UInt64, Int32, Int64, String, Bytes, } _reserved_words = { 'auto', 'else', 'long', 'switch', 'break', 'enum', 'register', 'typedef', 'case', 'extern', 'return', 'union', 'char', 'float', 'short', 'unsigned', 'const', 'for', 'signed', 'void', 'continue', 'goto', 'sizeof', 'volatile', 'default', 'if', 'static', 'while', 'do', 'int', 'struct', '_Packed', 'double', 'protocol', 'interface', 'implementation', 'NSObject', 'NSInteger', 'NSNumber', 'CGFloat', 'property', 'nonatomic', 'retain', 'strong', 'weak', 'unsafe_unretained', 'readwrite', 'description', 'id', 'delete', } _reserved_prefixes = { 'copy', 'new', } def fmt_obj(o): assert not isinstance(o, dict), "Only use for base type literals" if o is True: return 'true' if o is False: return 'false' if o is None: return 'nil' return pprint.pformat(o, width=1) def fmt_camel(name, upper_first=False, reserved=True): name = str(name) words = [word.capitalize() for word in split_words(name)] if not upper_first: words[0] = words[0].lower() ret = ''.join(words) if reserved: if ret.lower() in _reserved_words: ret += '_' # properties can't begin with certain keywords for reserved_prefix in _reserved_prefixes: if ret.lower().startswith(reserved_prefix): new_prefix = 'd' if not upper_first else 'D' ret = new_prefix + ret[0].upper() + ret[1:] continue return ret def fmt_enum_name(field_name, union): return 'DB{}{}{}'.format( fmt_class_caps(union.namespace.name), fmt_camel_upper(union.name), fmt_camel_upper(field_name)) def fmt_camel_upper(name, reserved=True): return fmt_camel(name, upper_first=True, reserved=reserved) def fmt_public_name(name): return fmt_camel_upper(name) def fmt_class(name): return fmt_camel_upper(name) def fmt_class_caps(name): return fmt_camel_upper(name).upper() def fmt_class_type(data_type, suppress_ptr=False): data_type, _ = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = '{}'.format(fmt_class_prefix(data_type)) else: result = _primitive_table.get(data_type.__class__, fmt_class(data_type.name)) if suppress_ptr: result = result.replace(' *', '') result = result.replace('*', '') if is_list_type(data_type): data_type, _ = unwrap_nullable(data_type.data_type) result = result + '<{}>'.format(fmt_type(data_type)) elif is_map_type(data_type): data_type, _ = unwrap_nullable(data_type.value_data_type) result = result + ''.format(fmt_type(data_type)) return result def fmt_func(name): return fmt_camel(name) def fmt_type(data_type, tag=False, has_default=False, no_ptr=False, is_prop=False): data_type, nullable = unwrap_nullable(data_type) if is_user_defined_type(data_type): base = '{}' if no_ptr else '{} *' result = base.format(fmt_class_prefix(data_type)) else: result = _primitive_table.get(data_type.__class__, fmt_class(data_type.name)) if is_list_type(data_type): data_type, _ = unwrap_nullable(data_type.data_type) base = '<{}>' if no_ptr else '<{}> *' result = result + base.format(fmt_type(data_type)) elif is_map_type(data_type): data_type, _ = unwrap_nullable(data_type.value_data_type) base = '' if no_ptr else ' *' result = result + base.format(fmt_type(data_type)) if tag: if (nullable or has_default) and not is_prop: result = 'nullable ' + result return result def fmt_route_type(data_type, tag=False, has_default=False): data_type, nullable = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = '{} *'.format(fmt_class_prefix(data_type)) else: result = _primitive_table_user_interface.get(data_type.__class__, fmt_class(data_type.name)) if is_list_type(data_type): data_type, _ = unwrap_nullable(data_type.data_type) result = result + '<{}> *'.format(fmt_type(data_type)) elif is_map_type(data_type): data_type, _ = unwrap_nullable(data_type.value_data_type) result = result + ''.format(fmt_type(data_type)) if is_user_defined_type(data_type) and tag: if nullable or has_default: result = 'nullable ' + result elif not is_void_type(data_type): result += '' return result def fmt_class_prefix(data_type): return 'DB{}{}'.format( fmt_class_caps(data_type.namespace.name), fmt_class(data_type.name)) def fmt_validator(data_type): return _validator_table.get(data_type.__class__, fmt_class(data_type.name)) def fmt_serial_obj(data_type): data_type, _ = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = fmt_serial_class(fmt_class_prefix(data_type)) else: result = _serial_table.get(data_type.__class__, fmt_class(data_type.name)) return result def fmt_serial_class(class_name): return '{}Serializer'.format(class_name) def fmt_route_obj_class(namespace_name): return 'DB{}RouteObjects'.format(fmt_class_caps(namespace_name)) def fmt_routes_class(namespace_name, auth_type): auth_type_to_use = auth_type if auth_type == 'noauth': auth_type_to_use = 'user' return 'DB{}{}AuthRoutes'.format( fmt_class_caps(namespace_name), fmt_camel_upper(auth_type_to_use)) def fmt_route_var(namespace_name, route): ret = 'DB{}{}'.format( fmt_class_caps(namespace_name), fmt_camel_upper(route.name)) if route.version != 1: ret = '{}V{}'.format(ret, route.version) return ret def fmt_route_func(route): ret = fmt_var(route.name) if route.version != 1: ret = '{}V{}'.format(ret, route.version) return ret def fmt_func_args(arg_str_pairs): result = [] first_arg = True for arg_name, arg_value in arg_str_pairs: if first_arg: result.append('{}'.format(arg_value)) first_arg = False else: result.append('{}:{}'.format(arg_name, arg_value)) return ' '.join(result) def fmt_func_args_declaration(arg_str_pairs): result = [] first_arg = True for arg_name, arg_type in arg_str_pairs: if first_arg: result.append('({}){}'.format(arg_type, arg_name)) first_arg = False else: result.append('{0}:({1}){0}'.format(arg_name, arg_type)) return ' '.join(result) def fmt_func_args_from_fields(args): result = [] first_arg = True for arg in args: if first_arg: result.append( '({}){}'.format(fmt_type(arg.data_type), fmt_var(arg.name))) first_arg = False else: result.append('{}:({}){}'.format( fmt_var(arg.name), fmt_type(arg.data_type), fmt_var(arg.name))) return ' '.join(result) def fmt_func_call(caller, callee, args=None): if args: result = '[{} {}:{}]'.format(caller, callee, args) else: result = '[{} {}]'.format(caller, callee) return result def fmt_alloc_call(caller): return '[{} alloc]'.format(caller) def fmt_default_value(field): if is_tag_ref(field.default): return '[[{} alloc] initWith{}]'.format( fmt_class_prefix(field.default.union_data_type), fmt_class(field.default.tag_name)) elif is_numeric_type(field.data_type): return '@({})'.format(field.default) elif is_boolean_type(field.data_type): if field.default: bool_str = 'YES' else: bool_str = 'NO' return '@{}'.format(bool_str) elif is_string_type(field.data_type): return '@"{}"'.format(field.default) else: raise TypeError( 'Can\'t handle default value type %r' % type(field.data_type)) def fmt_ns_number_call(data_type): result = '' if is_numeric_type(data_type): if isinstance(data_type, UInt32): result = 'numberWithUnsignedInt' elif isinstance(data_type, UInt64): result = 'numberWithUnsignedLong' elif isinstance(data_type, Int32): result = 'numberWithInt' elif isinstance(data_type, Int64): result = 'numberWithLong' elif isinstance(data_type, Float32): result = 'numberWithDouble' elif isinstance(data_type, Float64): result = 'numberWithDouble' elif is_boolean_type(data_type): result = 'numberWithBool' return result def fmt_signature(func, args, return_type='void', class_func=False): modifier = '-' if not class_func else '+' if args: result = '{} ({}){}:{}'.format(modifier, return_type, func, args) else: result = '{} ({}){}'.format(modifier, return_type, func) return result def is_primitive_type(data_type): data_type, _ = unwrap_nullable(data_type) return data_type.__class__ in _wrapper_primitives def fmt_var(name): return fmt_camel(name) def fmt_property(field): attrs = ['nonatomic', 'readonly'] data_type, nullable = unwrap_nullable(field.data_type) if is_string_type(data_type): attrs.append('copy') if nullable: attrs.append('nullable') base_string = '@property ({}) {}{};' return base_string.format(', '.join(attrs), fmt_type(field.data_type, tag=True, is_prop=True), fmt_var(field.name)) def fmt_import(header_file): return '#import "{}.h"'.format(header_file) def fmt_property_str(prop, typ, attrs=None): if not attrs: attrs = ['nonatomic', 'readonly'] base_string = '@property ({}) {} {};' return base_string.format(', '.join(attrs), typ, prop) def append_to_jazzy_category_dict(jazzy_dict, label, item): for category_dict in jazzy_dict['custom_categories']: if category_dict['name'] == label: category_dict['children'].append(item) return return None stone-3.3.1/stone/backends/obj_c_rsrc/000077500000000000000000000000001417406541500176645ustar00rootroot00000000000000stone-3.3.1/stone/backends/obj_c_rsrc/DBSerializableProtocol.h000066400000000000000000000021441417406541500243740ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import NS_ASSUME_NONNULL_BEGIN /// /// Protocol which all Obj-C SDK API route objects must implement, otherwise a compiler-warning /// is generated. /// @protocol DBSerializable /// /// Class method which returns a json-compatible dictionary representation of the /// supplied object. /// /// @param instance An instance of the API object to be serialized. /// /// @return A serialized, json-compatible dictionary representation of the API object. /// + (nullable NSDictionary *)serialize:(id)instance; /// /// Class method which returns an instantiation of the supplied object as represented /// by a json-compatible dictionary. /// /// @param dict A dictionary representation of the API object to be serialized. /// /// @return A deserialized, instantiation of the API object. /// + (id)deserialize:(NSDictionary *)dict; /// /// Description method. /// /// @return A human-readable representation of the current object. /// - (NSString *)description; @end NS_ASSUME_NONNULL_END stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneBase.h000066400000000000000000000045561417406541500221400ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import #import "DBSerializableProtocol.h" #import "DBStoneSerializers.h" NS_ASSUME_NONNULL_BEGIN /// /// Route object used to encapsulate route-specific information. /// @interface DBRoute : NSObject /// Name of the route. @property (nonatomic, readonly, copy) NSString *name; /// Namespace that the route is contained within. @property (nonatomic, readonly, copy) NSString *namespace_; /// Whether the route is deprecated. @property (nonatomic, readonly) NSNumber *deprecated; /// Class of the route's result object type (must implement `DBSerializable` /// protocol). @property (nonatomic, readonly, nullable) Class resultType; /// Class of the route's error object type (must implement `DBSerializable` /// protocol). Note: this class is only for route-specific errors, as opposed /// to more generic Dropbox API errors, as represented by the `DBRequestError` /// class. @property (nonatomic, readonly, nullable) Class errorType; /// Custom attributes associated with each route (can pertain to authentication /// type, host cluster, request-type, etc.). @property (nonatomic, readonly, nullable) NSDictionary *attrs; /// Serialization block for the route's result object type, if that result object /// type is an `NSArray`, otherwise nil. @property (nonatomic, readonly, nullable) id (^dataStructSerialBlock)(id dataStruct); /// Deserialization block for the route's result object type, if that result object /// type is a data structure, otherwise nil. @property (nonatomic, readonly, nullable) id (^dataStructDeserialBlock)(id dataStruct); /// Initializes the route object. - (nonnull instancetype)init:(NSString *)name namespace_:(NSString *)namespace_ deprecated:(NSNumber *)deprecated resultType:(nullable Class)resultType errorType:(nullable Class)errorType attrs:(NSDictionary *)attrs dataStructSerialBlock:(id (^_Nullable)(id))dataStructSerialBlock dataStructDeserialBlock:(id (^_Nullable)(id))dataStructDeserialBlock; @end /// /// Wrapper object designed to represent a nil response from the Dropbox API. /// @interface DBNilObject : NSObject @end NS_ASSUME_NONNULL_END stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneBase.m000066400000000000000000000016671417406541500221450ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import "DBStoneBase.h" @implementation DBRoute - (instancetype)init:(NSString *)name namespace_:(NSString *)namespace_ deprecated:(NSNumber *)deprecated resultType:(Class)resultType errorType:(Class)errorType attrs:(NSDictionary *)attrs dataStructSerialBlock:(id (^)(id))dataStructSerialBlock dataStructDeserialBlock:(id (^)(id))dataStructDeserialBlock { self = [self init]; if (self != nil) { _name = name; _namespace_ = namespace_; _deprecated = deprecated; _resultType = resultType; _errorType = errorType; _attrs = attrs; _dataStructSerialBlock = dataStructSerialBlock; _dataStructDeserialBlock = dataStructDeserialBlock; } return self; } @end @implementation DBNilObject @end stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneSerializers.h000066400000000000000000000062321417406541500235530ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import #import "DBSerializableProtocol.h" NS_ASSUME_NONNULL_BEGIN /// /// Category to ensure `NSArray` class "implements" `DBSerializable` protocol, which is /// required for all Obj-C SDK API route arguments. This avoids a compiler warning for /// `NSArray` route arguments. /// @interface NSArray (DBSerializable) + (nullable NSDictionary *)serialize:(id)obj; + (id)deserialize:(NSDictionary *)dict; @end /// /// Category to ensure `NSString` class "implements" `DBSerializable` protocol, which is /// required for all Obj-C SDK API route arguments. This avoids a compiler warning for /// `NSString` route arguments. /// @interface NSString (DBSerializable) + (nullable NSDictionary *)serialize:(id)obj; + (id)deserialize:(NSDictionary *)dict; @end /// /// Serializer functions used by the SDK to serialize/deserialize `NSDate` types. /// @interface DBNSDateSerializer : NSObject /// Returns a json-compatible `NSString` that represents an `NSDate` type based on the supplied /// `NSDate` object and date format string. + (NSString *)serialize:(NSDate *)value dateFormat:(NSString *)dateFormat; /// Returns an `NSDate` object from the supplied `NSString`-representation of an `NSDate` object and /// the supplied date format string. + (NSDate *)deserialize:(NSString *)value dateFormat:(NSString *)dateFormat; @end /// /// Serializer functions used by the SDK to serialize/deserialize `NSArray` types. /// @interface DBArraySerializer : NSObject /// Applies a serialization block to each element in the array and returns a new array with /// all elements serialized. The serialization block either serializes the object, or if the /// object is a wrapper for a primitive type, it leaves it unchanged. + (NSArray *)serialize:(NSArray *)value withBlock:(id (^_Nonnull)(id))serializeBlock; /// Applies a deserialization block to each element in the array and returns a new array with /// all elements deserialized. The serialization block either deserializes the object, or if the /// object is a wrapper for a primitive type, it leaves it unchanged. + (NSArray *)deserialize:(NSArray *)jsonData withBlock:(id (^_Nonnull)(id))deserializeBlock; @end /// /// Serializer functions used by the SDK to serialize/deserialize `NSArray` types. /// @interface DBMapSerializer : NSObject /// Applies a serialization block to each element in the map and returns a new map with /// all elements serialized. The serialization block either serializes the object, or if the /// object is a wrapper for a primitive type, it leaves it unchanged. + (NSDictionary *)serialize:(NSDictionary *)value withBlock:(id (^_Nonnull)(id))serializeBlock; /// Applies a deserialization block to each element in the map and returns a new map with /// all elements deserialized. The serialization block either deserializes the object, or if the /// object is a wrapper for a primitive type, it leaves it unchanged. + (NSDictionary *)deserialize:(NSDictionary *)jsonData withBlock:(id (^_Nonnull)(id))deserializeBlock; @end NS_ASSUME_NONNULL_END stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneSerializers.m000066400000000000000000000203501417406541500235550ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import "DBStoneSerializers.h" #import "DBStoneValidators.h" static NSDateFormatter *sFormatter = nil; static NSString *sDateFormat = nil; @implementation DBNSDateSerializer + (void)initialize { if (self == [DBNSDateSerializer class]) { sFormatter = [[NSDateFormatter alloc] init]; [sFormatter setTimeZone:[NSTimeZone timeZoneWithName:@"UTC"]]; [sFormatter setLocale:[NSLocale localeWithLocaleIdentifier:@"en_US_POSIX"]]; } } + (NSString *)serialize:(NSDate *)value dateFormat:(NSString *)dateFormat { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } @synchronized (sFormatter) { if (![dateFormat isEqualToString:sDateFormat]) { [sFormatter setDateFormat:[self convertFormat:dateFormat]]; sDateFormat = [dateFormat copy]; } return [sFormatter stringFromDate:value]; } } + (NSDate *)deserialize:(NSString *)value dateFormat:(NSString *)dateFormat { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } @synchronized (sFormatter) { if (![dateFormat isEqualToString:sDateFormat]) { [sFormatter setDateFormat:[self convertFormat:dateFormat]]; sDateFormat = [dateFormat copy]; } return [sFormatter dateFromString:value]; } } + (NSString *)formatDateToken:(NSString *)token { NSString *result = @""; if ([token isEqualToString:@"%a"]) { // Weekday as locale's abbreviated name. result = @"EEE"; } else if ([token isEqualToString:@"%A"]) { // Weekday as locale's full name. result = @"EEE"; } else if ([token isEqualToString:@"%w"]) { // Weekday as a decimal number, where 0 is Sunday and 6 is Saturday. 0, 1, // ..., 6 result = @"ccccc"; } else if ([token isEqualToString:@"%d"]) { // Day of the month as a zero-padded decimal number. 01, 02, ..., 31 result = @"dd"; } else if ([token isEqualToString:@"%b"]) { // Month as locale's abbreviated name. result = @"MMM"; } else if ([token isEqualToString:@"%B"]) { // Month as locale's full name. result = @"MMMM"; } else if ([token isEqualToString:@"%m"]) { // Month as a zero-padded decimal number. 01, 02, ..., 12 result = @"MM"; } else if ([token isEqualToString:@"%y"]) { // Year without century as a zero-padded decimal number. 00, 01, ..., 99 result = @"yy"; } else if ([token isEqualToString:@"%Y"]) { // Year with century as a decimal number. 1970, 1988, 2001, 2013 result = @"yyyy"; } else if ([token isEqualToString:@"%H"]) { // Hour (24-hour clock) as a zero-padded decimal number. 00, 01, ..., 23 result = @"HH"; } else if ([token isEqualToString:@"%I"]) { // Hour (12-hour clock) as a zero-padded decimal number. 01, 02, ..., 12 result = @"hh"; } else if ([token isEqualToString:@"%p"]) { // Locale's equivalent of either AM or PM. result = @"a"; } else if ([token isEqualToString:@"%M"]) { // Minute as a zero-padded decimal number. 00, 01, ..., 59 result = @"mm"; } else if ([token isEqualToString:@"%S"]) { // Second as a zero-padded decimal number. 00, 01, ..., 59 result = @"ss"; } else if ([token isEqualToString:@"%f"]) { // Microsecond as a decimal number, zero-padded on the left. 000000, // 000001, ..., 999999 result = @"SSSSSS"; } else if ([token isEqualToString:@"%z"]) { // UTC offset in the form +HHMM or -HHMM (empty string if the the object // is naive). (empty), +0000, -0400, +1030 result = @"Z"; } else if ([token isEqualToString:@"%Z"]) { // Time zone name (empty string if the object is naive). (empty), UTC, // EST, CST result = @"z"; } else if ([token isEqualToString:@"%j"]) { // Day of the year as a zero-padded decimal number. 001, 002, ..., 366 result = @"DDD"; } else if ([token isEqualToString:@"%U"]) { // Week number of the year (Sunday as the first day of the week) as a zero // padded decimal number. All days in a new year preceding the first // Sunday are considered to be in week 0. 00, 01, ..., 53 (6) result = @"ww"; } else if ([token isEqualToString:@"%W"]) { // Week number of the year (Monday as the first day of the week) as a // decimal number. All days in a new year preceding the first Monday are // considered to be in week 0. 00, 01, ..., 53 (6) result = @"ww"; } else if ([token isEqualToString:@"%c"]) { // Locale's appropriate date and time representation. result = @""; // unsupported } else if ([token isEqualToString:@"%x"]) { // Locale's appropriate date representation. result = @""; // unsupported } else if ([token isEqualToString:@"%X"]) { // Locale's appropriate time representation. result = @""; // unsupported } else if ([token isEqualToString:@"%%"]) { // A literal '%' character. result = @""; } else if ([token isEqualToString:@"%"]) { result = @""; } else { result = @""; } return result; } + (NSString *)convertFormat:(NSString *)format { NSCharacterSet *alphabeticSet = [NSCharacterSet characterSetWithCharactersInString:@"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"]; NSMutableString *newFormat = [@"" mutableCopy]; BOOL inQuotedText = NO; NSUInteger len = [format length]; NSUInteger i = 0; while (i < len) { char ch = (char)[format characterAtIndex:i]; if (ch == '%') { if (i >= len - 1) { return nil; } i++; ch = (char)[format characterAtIndex:i]; NSString *token = [NSString stringWithFormat:@"%%%c", ch]; if (inQuotedText) { [newFormat appendString:@"'"]; inQuotedText = NO; } [newFormat appendString:[self formatDateToken:token]]; } else { if ([alphabeticSet characterIsMember:ch]) { if (!inQuotedText) { [newFormat appendString:@"'"]; inQuotedText = YES; } } else if (ch == '\'') { [newFormat appendString:@"'"]; } [newFormat appendString:[NSString stringWithFormat:@"%c", ch]]; } i++; } if (inQuotedText) { [newFormat appendString:@"'"]; } return newFormat; } @end @implementation DBArraySerializer + (NSArray *)serialize:(NSArray *)value withBlock:(id (^)(id))serializeBlock { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } NSMutableArray *resultArray = [[NSMutableArray alloc] init]; for (id element in value) { [resultArray addObject:serializeBlock(element)]; } return resultArray; } + (NSArray *)deserialize:(NSArray *)value withBlock:(id (^)(id))deserializeBlock { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } NSMutableArray *resultArray = [[NSMutableArray alloc] init]; for (id element in value) { [resultArray addObject:deserializeBlock(element)]; } return resultArray; } @end @implementation DBMapSerializer + (NSDictionary *)serialize:(NSDictionary *)value withBlock:(id (^)(id))serializeBlock { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } NSMutableDictionary *resultDict = [[NSMutableDictionary alloc] init]; for (id key in value) { [resultDict setObject:serializeBlock(value[key]) forKey:key]; } return resultDict; } + (NSDictionary *)deserialize:(NSDictionary *)value withBlock:(id (^)(id))deserializeBlock { if (value == nil) { [DBStoneValidators raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } NSMutableDictionary *resultDict = [[NSMutableDictionary alloc] init]; for (id key in value) { [resultDict setObject:deserializeBlock(value[key]) forKey:key]; } return resultDict; } @end stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneValidators.h000066400000000000000000000035741417406541500233750ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import NS_ASSUME_NONNULL_BEGIN /// /// Validator functions used by SDK to impose value constraints. /// @interface DBStoneValidators : NSObject /// Validator for `NSString` objects. Enforces minimum length and/or maximum length and/or regex pattern. + (void (^_Nonnull)(NSString *))stringValidator:(nullable NSNumber *)minLength maxLength:(nullable NSNumber *)maxLength pattern:(nullable NSString *)pattern; /// Validator for `NSNumber` objects. Enforces minimum value and/or maximum value. + (void (^_Nonnull)(NSNumber *))numericValidator:(nullable NSNumber *)minValue maxValue:(nullable NSNumber *)maxValue; /// Validator for `NSArray` objects. Enforces minimum number of items and/or maximum minimum number of items. Method /// requires a validator block that can validate each item in the array. + (void (^_Nonnull)(NSArray *))arrayValidator:(nullable NSNumber *)minItems maxItems:(nullable NSNumber *)maxItems itemValidator:(void (^_Nullable)(T))itemValidator; /// Validator for `NSDictionary` objects. Enforces minimum number of items and/or maximum minimum number of items. /// Method /// requires a validator block that can validate each item in the array. + (void (^_Nonnull)(NSDictionary *))mapValidator:(void (^_Nullable)(T))itemValidator; /// Wrapper validator for nullable objects. Maintains a reference to the object's normal non-nullable validator. + (void (^_Nonnull)(T))nullableValidator:(void (^_Nonnull)(T))internalValidator; + (void (^_Nonnull)(id))nonnullValidator:(void (^_Nullable)(id))internalValidator; + (void)raiseIllegalStateErrorWithMessage:(NSString *)message; @end NS_ASSUME_NONNULL_END stone-3.3.1/stone/backends/obj_c_rsrc/DBStoneValidators.m000066400000000000000000000114241417406541500233730ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// #import "DBStoneValidators.h" @implementation DBStoneValidators + (void (^)(NSString *))stringValidator:(NSNumber *)minLength maxLength:(NSNumber *)maxLength pattern:(NSString *)pattern { void (^validator)(NSString *) = ^(NSString *value) { if (minLength != nil) { if ([value length] < [minLength unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at least %@ characters", [minLength stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } if (maxLength != nil) { if ([value length] > [maxLength unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at most %@ characters", [minLength stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } if (pattern != nil && pattern.length != 0) { NSError *error; NSRegularExpression *re = [NSRegularExpression regularExpressionWithPattern:pattern options:0 error:&error]; NSArray *matches = [re matchesInString:value options:0 range:NSMakeRange(0, [value length])]; if ([matches count] == 0) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must match pattern \"%@\"", [re pattern]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } }; return validator; } + (void (^)(NSNumber *))numericValidator:(NSNumber *)minValue maxValue:(NSNumber *)maxValue { void (^validator)(NSNumber *) = ^(NSNumber *value) { if (minValue != nil) { if ([value unsignedIntegerValue] < [minValue unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at least %@", [minValue stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } if (maxValue != nil) { if ([value unsignedIntegerValue] > [maxValue unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at most %@", [maxValue stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } }; return validator; } + (void (^)(NSArray *))arrayValidator:(NSNumber *)minItems maxItems:(NSNumber *)maxItems itemValidator:(void (^)(id))itemValidator { void (^validator)(NSArray *) = ^(NSArray *value) { if (minItems != nil) { if ([value count] < [minItems unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at least %@ items", [minItems stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } if (maxItems != nil) { if ([value count] > [maxItems unsignedIntegerValue]) { NSString *exceptionMessage = [NSString stringWithFormat:@"value must be at most %@ items", [maxItems stringValue]]; [[self class] raiseIllegalStateErrorWithMessage:exceptionMessage]; } } if (itemValidator != nil) { for (id item in value) { itemValidator(item); } } }; return validator; } + (void (^)(NSDictionary *))mapValidator:(void (^)(id))itemValidator { void (^validator)(NSDictionary *) = ^(NSDictionary *value) { if (itemValidator != nil) { for (id key in value) { itemValidator(value[key]); } } }; return validator; } + (void (^)(id))nullableValidator:(void (^)(id))internalValidator { void (^validator)(id) = ^(id value) { if (value != nil) { internalValidator(value); } }; return validator; } + (void (^)(id))nonnullValidator:(void (^)(id))internalValidator { void (^validator)(id) = ^(id value) { if (value == nil) { [[self class] raiseIllegalStateErrorWithMessage:@"Value must not be `nil`"]; } if (internalValidator != nil) { internalValidator(value); } }; return validator; } + (void)raiseIllegalStateErrorWithMessage:(NSString *)message { NSString *exceptionMessage = [NSString stringWithFormat:@"%@:\n%@", message, [[NSThread callStackSymbols] objectAtIndex:0]]; [NSException raise:@"IllegalStateException" format:exceptionMessage, nil]; } @end stone-3.3.1/stone/backends/obj_c_types.py000066400000000000000000002131361417406541500204370ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json import os import shutil import six from stone.ir import ( is_list_type, is_map_type, is_nullable_type, is_numeric_type, is_string_type, is_struct_type, is_timestamp_type, is_union_type, is_user_defined_type, is_void_type, unwrap_nullable, ) from stone.backends.obj_c_helpers import ( append_to_jazzy_category_dict, fmt_alloc_call, fmt_camel, fmt_camel_upper, fmt_class, fmt_class_prefix, fmt_class_type, fmt_default_value, fmt_enum_name, fmt_func, fmt_func_args, fmt_func_args_declaration, fmt_func_args_from_fields, fmt_func_call, fmt_import, fmt_property, fmt_property_str, fmt_public_name, fmt_routes_class, fmt_route_func, fmt_route_obj_class, fmt_route_var, fmt_serial_class, fmt_serial_obj, fmt_signature, fmt_type, fmt_validator, fmt_var, is_primitive_type, ) from stone.backends.obj_c import ( base_file_comment, comment_prefix, ObjCBaseBackend, undocumented, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any _cmdline_parser = argparse.ArgumentParser(prog='obj-c-types-backend') _cmdline_parser.add_argument( '-r', '--route-method', help=('A string used to construct the location of an Objective-C method for a ' 'given route; use {ns} as a placeholder for namespace name and ' '{route} for the route name.'), ) _cmdline_parser.add_argument( '-d', '--documentation', action='store_true', help=('Sets whether documentation is generated.'), ) _cmdline_parser.add_argument( '-e', '--exclude-from-analysis', action='store_true', help=('Sets whether generated code should marked for exclusion ' + 'from analysis.'), ) class ObjCTypesBackend(ObjCBaseBackend): """Generates Obj C modules to represent the input Stone spec.""" cmdline_parser = _cmdline_parser obj_name_to_namespace = {} # type: typing.Dict[str, str] namespace_to_has_route_auth_list = {} # type: typing.Dict[typing.Any, typing.Set] def generate(self, api): """ Generates a module for each namespace. Each namespace will have Obj C classes to represent data types and routes in the Stone spec. """ rsrc_folder = os.path.join(os.path.dirname(__file__), 'obj_c_rsrc') rsrc_output_folder = os.path.join(self.target_folder_path, 'Resources') if not os.path.exists(rsrc_output_folder): os.makedirs(rsrc_output_folder) self.logger.info('Copying DBStoneValidators.{h,m} to output folder') shutil.copy( os.path.join(rsrc_folder, 'DBStoneValidators.h'), rsrc_output_folder) shutil.copy( os.path.join(rsrc_folder, 'DBStoneValidators.m'), rsrc_output_folder) self.logger.info('Copying DBStoneSerializers.{h,m} to output folder') shutil.copy( os.path.join(rsrc_folder, 'DBStoneSerializers.h'), rsrc_output_folder) shutil.copy( os.path.join(rsrc_folder, 'DBStoneSerializers.m'), rsrc_output_folder) self.logger.info('Copying DBStoneBase.{h,m} to output folder') shutil.copy( os.path.join(rsrc_folder, 'DBStoneBase.h'), rsrc_output_folder) shutil.copy( os.path.join(rsrc_folder, 'DBStoneBase.m'), rsrc_output_folder) self.logger.info('Copying DBSerializableProtocol.h to output folder') shutil.copy( os.path.join(rsrc_folder, 'DBSerializableProtocol.h'), rsrc_output_folder) jazzy_cfg = None if self.args.documentation: jazzy_cfg_path = os.path.join('../Format', 'jazzy.json') with open(jazzy_cfg_path, encoding='utf-8') as jazzy_file: jazzy_cfg = json.load(jazzy_file) for idx, namespace in enumerate(api.namespaces.values()): ns_name = fmt_public_name(namespace.name) ns_dict = {"name": ns_name, "children": [], } jazzy_cfg['custom_categories'].insert(idx, ns_dict) for namespace in api.namespaces.values(): self.namespace_to_has_route_auth_list[namespace] = set() if namespace.routes: for route in namespace.routes: auth_types = set(map(lambda x: x.strip(), route.attrs.get('auth').split(','))) if 'noauth' not in auth_types: self.namespace_to_has_route_auth_list[namespace].add( route.attrs.get('auth')) else: self.namespace_to_has_route_auth_list[namespace].add( 'user') with self.output_to_relative_path('DBSDKImportsGenerated.h'): self._generate_all_imports(api) for namespace in api.namespaces.values(): for data_type in namespace.linearize_data_types(): self.obj_name_to_namespace[data_type.name] = fmt_class_prefix( data_type) for namespace in api.namespaces.values(): ns_name = fmt_public_name(namespace.name) self._generate_namespace_types(namespace, jazzy_cfg) if namespace.routes: if self.args.documentation: for auth_type in self.namespace_to_has_route_auth_list[ namespace]: append_to_jazzy_category_dict( jazzy_cfg, 'Routes', fmt_routes_class(ns_name, auth_type)) append_to_jazzy_category_dict( jazzy_cfg, 'RouteObjects', fmt_route_obj_class(ns_name)) self._generate_route_objects_m(api.route_schema, namespace) self._generate_route_objects_h(api.route_schema, namespace) if self.args.documentation: with self.output_to_relative_path('../../../../.jazzy.json'): self.emit_raw(json.dumps(jazzy_cfg, indent=2) + '\n') def _generate_all_imports(self, api): self.emit_raw(base_file_comment) self.emit('/// Import autogenerated files') self.emit() self.emit('// Routes') for namespace in api.namespaces.values(): if namespace.routes: for auth_type in self.namespace_to_has_route_auth_list[ namespace]: self.emit( fmt_import( fmt_routes_class(namespace.name, auth_type))) self.emit(fmt_import(fmt_route_obj_class(namespace.name))) self.emit() for namespace in api.namespaces.values(): namespace_imports = [] self.emit() self.emit( '// `{}` namespace types'.format(fmt_class(namespace.name))) self.emit() for data_type in namespace.linearize_data_types(): namespace_imports.append(fmt_class_prefix(data_type)) self._generate_imports_m(namespace_imports) def _generate_namespace_types(self, namespace, jazzy_cfg): """Creates Obj C argument, error, serializer and deserializer types for the given namespace.""" ns_name = fmt_public_name(namespace.name) output_path = os.path.join('ApiObjects', ns_name) output_path_headers = os.path.join(output_path, 'Headers') for data_type in namespace.linearize_data_types(): class_name = fmt_class_prefix(data_type) if self.args.documentation: append_to_jazzy_category_dict(jazzy_cfg, ns_name, class_name) append_to_jazzy_category_dict( jazzy_cfg, 'Serializers', '{}Serializer'.format(class_name)) if is_struct_type(data_type): # struct header file_path = os.path.join(output_path_headers, class_name + '.h') with self.output_to_relative_path(file_path): self.emit_raw(base_file_comment) self._generate_struct_class_h(data_type) elif is_union_type(data_type): if self.args.documentation: append_to_jazzy_category_dict( jazzy_cfg, 'Tags', '{}Tag'.format(fmt_class_prefix(data_type))) # union header file_path = os.path.join(output_path_headers, class_name + '.h') with self.output_to_relative_path(file_path): self.emit_raw(base_file_comment) self._generate_union_class_h(data_type) else: raise TypeError('Can\'t handle type %r' % type(data_type)) file_path = os.path.join( output_path, 'DB{}Objects.m'.format(fmt_camel_upper(namespace.name))) with self.output_to_relative_path(file_path): self.emit_raw(base_file_comment) description = '/// Arguments, results, and errors for the `{}` namespace.'.format( fmt_camel_upper(namespace.name)) self.emit(description) if self.args.exclude_from_analysis: self.emit() self.emit('#ifndef __clang_analyzer__') for data_type in namespace.linearize_data_types(): if is_struct_type(data_type): # struct implementation self._generate_struct_class_m(data_type) elif is_union_type(data_type): # union implementation self._generate_union_class_m(data_type) if self.args.exclude_from_analysis: self.emit('#endif') def _generate_struct_class_m(self, struct): """Defines an Obj C implementation file that represents a struct in Stone.""" self.emit() self._generate_imports_m( self._get_imports_m( struct, default_imports=['DBStoneSerializers', 'DBStoneValidators'])) struct_name = fmt_class_prefix(struct) self.emit('#pragma mark - API Object') self.emit() with self.block_m(struct_name): self.emit('#pragma mark - Constructors') self.emit() self._generate_struct_cstor(struct) self._generate_struct_cstor_default(struct) self.emit('#pragma mark - Serialization methods') self.emit() self._generate_serializable_funcs(struct_name) self.emit('#pragma mark - Debug Description method') self.emit() self._generate_debug_description_func(struct_name) self.emit('#pragma mark - Copyable method') self.emit() self._generate_copyable_func() self.emit('#pragma mark - Hash method') self.emit() self._generate_hash_func(struct) self.emit('#pragma mark - Equality method') self.emit() self._generate_equality_func(struct) self.emit() self.emit() self.emit('#pragma mark - Serializer Object') self.emit() with self.block_m(fmt_serial_class(struct_name)): self._generate_struct_serializer(struct) self._generate_struct_deserializer(struct) def _generate_struct_class_h(self, struct): """Defines an Obj C header file that represents a struct in Stone.""" self._generate_init_imports_h(struct) self._generate_imports_h(self._get_imports_h(struct)) self.emit() self.emit('NS_ASSUME_NONNULL_BEGIN') self.emit() self.emit('#pragma mark - API Object') self.emit() self._generate_class_comment(struct) struct_name = fmt_class_prefix(struct) with self.block_h_from_data_type(struct, protocol=['DBSerializable', 'NSCopying']): self.emit('#pragma mark - Instance fields') self.emit() self._generate_struct_properties(struct.fields) self.emit('#pragma mark - Constructors') self.emit() self._generate_struct_cstor_signature(struct) self._generate_struct_cstor_signature_default(struct) self._generate_init_unavailable_signature(struct) self.emit() self.emit() self.emit('#pragma mark - Serializer Object') self.emit() self.emit(comment_prefix) self.emit_wrapped_text( 'The serialization class for the `{}` struct.'.format( fmt_class(struct.name)), prefix=comment_prefix) self.emit(comment_prefix) with self.block_h(fmt_serial_class(struct_name)): self._generate_serializer_signatures(struct_name) self.emit() self.emit('NS_ASSUME_NONNULL_END') self.emit() def _generate_union_class_m(self, union): """Defines an Obj C implementation file that represents a union in Stone.""" self.emit() self._generate_imports_m( self._get_imports_m( union, default_imports=['DBStoneSerializers', 'DBStoneValidators'])) union_name = fmt_class_prefix(union) self.emit('#pragma mark - API Object') self.emit() with self.block_m(fmt_class_prefix(union)): self._generate_synthesize_ivars(union) self.emit('#pragma mark - Constructors') self.emit() self._generate_union_cstor_funcs(union) self.emit('#pragma mark - Instance field accessors') self.emit() self._generate_union_tag_vars_funcs(union) self.emit('#pragma mark - Tag state methods') self.emit() self._generate_union_tag_state_funcs(union) self.emit('#pragma mark - Serialization methods') self.emit() self._generate_serializable_funcs(union_name) self.emit('#pragma mark - Debug Description method') self.emit() self._generate_debug_description_func(union_name) self.emit('#pragma mark - Copyable method') self.emit() self._generate_copyable_func() self.emit('#pragma mark - Hash method') self.emit() self._generate_hash_func(union) self.emit('#pragma mark - Equality method') self.emit() self._generate_equality_func(union) self.emit() self.emit() self.emit('#pragma mark - Serializer Object') self.emit() with self.block_m(fmt_serial_class(union_name)): self._generate_union_serializer(union) self._generate_union_deserializer(union) def _generate_union_class_h(self, union): """Defines an Obj C header file that represents a union in Stone.""" self._generate_init_imports_h(union) self._generate_imports_h(self._get_imports_h(union)) self.emit() self.emit('NS_ASSUME_NONNULL_BEGIN') self.emit() self.emit('#pragma mark - API Object') self.emit() self._generate_class_comment(union) union_name = fmt_class_prefix(union) with self.block_h_from_data_type(union, protocol=['DBSerializable', 'NSCopying']): self.emit('#pragma mark - Instance fields') self.emit() self._generate_union_tag_state(union) self._generate_union_tag_property(union) self._generate_union_properties(union.all_fields) self.emit('#pragma mark - Constructors') self.emit() self._generate_union_cstor_signatures(union, union.all_fields) self._generate_init_unavailable_signature(union) self.emit('#pragma mark - Tag state methods') self.emit() self._generate_union_tag_access_signatures(union) self.emit() self.emit() self.emit('#pragma mark - Serializer Object') self.emit() self.emit(comment_prefix) self.emit_wrapped_text( 'The serialization class for the `{}` union.'.format(union_name), prefix=comment_prefix) self.emit(comment_prefix) with self.block_h(fmt_serial_class(union_name)): self._generate_serializer_signatures(union_name) self.emit() self.emit('NS_ASSUME_NONNULL_END') self.emit() def _generate_synthesize_ivars(self, union): non_void_exists = False for field in union.all_fields: if not is_void_type(field.data_type): non_void_exists = True self.emit('@synthesize {} = _{};'.format( fmt_var(field.name), fmt_var(field.name))) if non_void_exists: self.emit() def _generate_struct_cstor(self, struct): """Emits struct standard constructor.""" with self.block_func( func=self._cstor_name_from_fields(struct.all_fields), args=fmt_func_args_from_fields(struct.all_fields), return_type='instancetype'): for field in struct.all_fields: self._generate_validator(field) self.emit() super_fields = [ f for f in struct.all_fields if f not in struct.fields ] if super_fields: super_args = fmt_func_args([(fmt_var(f.name), fmt_var(f.name)) for f in super_fields]) self.emit('self = [super {}:{}];'.format( self._cstor_name_from_fields(super_fields), super_args)) else: if struct.parent_type: self.emit('self = [super initDefault];') else: self.emit('self = [super init];') with self.block_init(): for field in struct.fields: field_name = fmt_var(field.name) if field.has_default: self.emit('_{0} = {0} ?: {1};'.format( field_name, fmt_default_value(field))) else: self.emit('_{0} = {0};'.format(field_name)) self.emit() def _generate_struct_cstor_default(self, struct): """Emits struct convenience constructor. Default arguments are omitted.""" if not self._struct_has_defaults(struct): return fields_no_default = [ f for f in struct.all_fields if not f.has_default and not is_nullable_type(f.data_type) ] with self.block_func( func=self._cstor_name_from_fields(fields_no_default), args=fmt_func_args_from_fields(fields_no_default), return_type='instancetype'): args = ([(fmt_var(f.name), fmt_var(f.name) if not f.has_default and not is_nullable_type(f.data_type) else 'nil') for f in struct.all_fields]) cstor_args = fmt_func_args(args) self.emit('return [self {}:{}];'.format( self._cstor_name_from_fields(struct.all_fields), cstor_args)) self.emit() def _generate_struct_cstor_signature(self, struct): """Emits struct standard constructor signature to be used in the struct's header file.""" fields = struct.all_fields self.emit(comment_prefix) description_str = 'Full constructor for the struct (exposes all instance variables).' self.emit_wrapped_text(description_str, prefix=comment_prefix) signature = fmt_signature( func=self._cstor_name_from_fields(fields), args=self._cstor_args_from_fields(fields, is_struct=True), return_type='instancetype') self.emit(comment_prefix) for field in struct.all_fields: doc = self.process_doc(field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text( '@param {} {}'.format(fmt_var(field.name), doc), prefix=comment_prefix) if struct.all_fields: self.emit(comment_prefix) self.emit_wrapped_text( '@return An initialized instance.', prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(signature)) self.emit() def _generate_struct_cstor_signature_default(self, struct): """Emits struct convenience constructor with default arguments ommitted signature to be used in the struct header file.""" if not self._struct_has_defaults(struct): return fields_no_default = [ f for f in struct.all_fields if not f.has_default and not is_nullable_type(f.data_type) ] signature = fmt_signature( func=self._cstor_name_from_fields(fields_no_default), args=self._cstor_args_from_fields( fields_no_default, is_struct=True), return_type='instancetype') self.emit(comment_prefix) description_str = ( 'Convenience constructor (exposes only non-nullable ' 'instance variables with no default value).') self.emit_wrapped_text(description_str, prefix=comment_prefix) self.emit(comment_prefix) for field in fields_no_default: doc = self.process_doc(field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text( '@param {} {}'.format(fmt_var(field.name), doc), prefix=comment_prefix) if struct.all_fields: self.emit(comment_prefix) self.emit_wrapped_text( '@return An initialized instance.', prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(signature)) self.emit() def _generate_union_cstor_funcs(self, union): """Emits standard union constructor.""" for field in union.all_fields: enum_field_name = fmt_enum_name(field.name, union) func_args = [] if is_void_type( field.data_type) else fmt_func_args_from_fields([field]) with self.block_func( func=self._cstor_name_from_field(field), args=func_args, return_type='instancetype'): self.emit('self = [super init];') with self.block_init(): self.emit('_tag = {};'.format(enum_field_name)) if not is_void_type(field.data_type): self.emit('_{} = {};'.format( fmt_var(field.name), fmt_var(field.name))) self.emit() def _generate_union_cstor_signatures(self, union, fields): # pylint: disable=unused-argument """Emits union constructor signatures to be used in the union's header file.""" for field in fields: args = self._cstor_args_from_fields( [field] if not is_void_type(field.data_type) else []) signature = fmt_signature( func=self._cstor_name_from_field(field), args=args, return_type='instancetype') self.emit(comment_prefix) self.emit_wrapped_text( 'Initializes union class with tag state of "{}".'.format( field.name), prefix=comment_prefix) self.emit(comment_prefix) if field.doc: doc = self.process_doc( field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text( 'Description of the "{}" tag state: {}'.format( field.name, doc), prefix=comment_prefix) self.emit(comment_prefix) if not is_void_type(field.data_type): doc = self.process_doc( field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text( '@param {} {}'.format(fmt_var(field.name), doc), prefix=comment_prefix) self.emit(comment_prefix) self.emit_wrapped_text( '@return An initialized instance.', prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(signature)) self.emit() def _generate_union_tag_state(self, union): """Emits union tag enum type, which stores union state.""" union_name = fmt_class_prefix(union) tag_type = fmt_enum_name('tag', union) description_str = ('The `{}` enum type represents the possible tag ' 'states with which the `{}` union can exist.') self.emit_wrapped_text( description_str.format(tag_type, union_name), prefix=comment_prefix) with self.block( 'typedef NS_CLOSED_ENUM(NSInteger, {})'.format(tag_type), after=';'): for field in union.all_fields: doc = self.process_doc( field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text(doc, prefix=comment_prefix) self.emit('{},'.format(fmt_enum_name(field.name, union))) self.emit() self.emit() def _generate_init_unavailable_signature(self, data_type): if not data_type.parent_type or is_union_type(data_type): self.emit('- (instancetype)init NS_UNAVAILABLE;') self.emit() def _generate_serializable_funcs(self, data_type_name): """Emits the two struct/union functions that implement the Serializable protocol.""" with self.block_func( func='serialize', args=fmt_func_args_declaration([('instance', 'id')]), return_type='nullable NSDictionary *', class_func=True): func_call = fmt_func_call( caller=fmt_serial_class(data_type_name), callee='serialize', args=fmt_func_args([('instance', 'instance')])) self.emit('return {};'.format(func_call)) self.emit() with self.block_func( func='deserialize', args=fmt_func_args_declaration([('dict', 'NSDictionary *')]), return_type='id', class_func=True): self.emit('return {};'.format( fmt_func_call( caller=fmt_serial_class(data_type_name), callee='deserialize', args=fmt_func_args([('dict', 'dict')])))) self.emit() def _generate_serializer_signatures(self, obj_name): """Emits the signatures of the serializer object's serializing functions.""" serial_signature = fmt_signature( func='serialize', args=fmt_func_args_declaration([( 'instance', '{} *'.format(obj_name))]), return_type='nullable NSDictionary *', class_func=True) deserial_signature = fmt_signature( func='deserialize', args=fmt_func_args_declaration([('dict', 'NSDictionary *')]), return_type='{} *'.format(obj_name), class_func=True) self.emit(comment_prefix) self.emit_wrapped_text( 'Serializes `{}` instances.'.format(obj_name), prefix=comment_prefix) self.emit(comment_prefix) self.emit_wrapped_text( '@param instance An instance of the `{}` API object.'.format( obj_name), prefix=comment_prefix) self.emit(comment_prefix) description_str = ('@return A json-compatible dictionary ' 'representation of the `{}` API object.') self.emit_wrapped_text( description_str.format(obj_name), prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(serial_signature)) self.emit() self.emit(comment_prefix) self.emit_wrapped_text( 'Deserializes `{}` instances.'.format(obj_name), prefix=comment_prefix) self.emit(comment_prefix) description_str = ('@param dict A json-compatible dictionary ' 'representation of the `{}` API object.') self.emit_wrapped_text( description_str.format(obj_name), prefix=comment_prefix) self.emit(comment_prefix) self.emit_wrapped_text( '@return An instantiation of the `{}` object.'.format(obj_name), prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(deserial_signature)) self.emit() def _generate_debug_description_func(self, data_type_name): with self.block_func( func='debugDescription', args=[], return_type='NSString *'): serialize_call = fmt_func_call( caller=fmt_serial_class(data_type_name), callee='serialize', args=fmt_func_args([('valueObj', 'self')])) self.emit('return {};'.format( fmt_func_call(caller=serialize_call, callee='description'))) self.emit() def _generate_copyable_func(self): with self.block_func( func='copyWithZone', args=fmt_func_args_declaration( [('zone', 'NSZone *')]), return_type='instancetype'): self.emit('#pragma unused(zone)') self.emit_wrapped_text('object is immutable', prefix=comment_prefix) self.emit('return self;') self.emit() def _generate_hash_func(self, data_type): with self.block_func( func='hash', return_type='NSUInteger'): self.emit('NSUInteger prime = 31;') self.emit('NSUInteger result = 1;') self.emit() if is_union_type(data_type): with self.block('switch (_tag)'): for field in data_type.all_fields: enum_field_name = fmt_enum_name(field.name, data_type) self.emit('case {}:'.format(enum_field_name)) self._generate_hash_func_helper(data_type, field) self.emit('break;') elif is_struct_type(data_type): for field in data_type.all_fields: self._generate_hash_func_helper(data_type, field) self.emit() self.emit('return prime * result;') self.emit() def _generate_hash_func_helper(self, data_type, field): _, nullable = unwrap_nullable(field.data_type) if nullable: with self.block('if (self.{} != nil)'.format(fmt_var(field.name))): self._generate_hash_check(data_type, field) else: self._generate_hash_check(data_type, field) def _generate_hash_check(self, data_type, field): if is_union_type(data_type) and is_void_type(field.data_type): self.emit('result = prime * result + [[self tagName] hash];') else: self.emit('result = prime * result + [self.{} hash];'.format(fmt_var(field.name))) def _generate_equality_func(self, data_type): is_equal_func_name = 'isEqualTo{}'.format(fmt_camel_upper(data_type.name)) with self.block_func(func='isEqual', args=fmt_func_args_declaration([('other', 'id')]), return_type='BOOL'): with self.block('if (other == self)'): self.emit('return YES;') with self.block('if (!other || ![other isKindOfClass:[self class]])'): self.emit('return NO;') self.emit('return [self {}:other];'.format(is_equal_func_name)) self.emit() if data_type.name[0].lower() in 'aeiou': other_obj_name = 'an{}'.format(fmt_class(data_type.name)) else: other_obj_name = 'a{}'.format(fmt_class(data_type.name)) with self.block_func( func=is_equal_func_name, args=fmt_func_args_declaration([(other_obj_name, '{} *'.format( fmt_class_type(data_type)))]), return_type='BOOL'): with self.block('if (self == {})'.format(other_obj_name)): self.emit('return YES;') if is_union_type(data_type): with self.block('if (self.tag != {}.tag)'.format(other_obj_name)): self.emit('return NO;') with self.block('switch (_tag)'): for field in data_type.all_fields: self._generate_equality_func_helper(data_type, field, other_obj_name) self.emit('return YES;') elif is_struct_type(data_type): for field in data_type.all_fields: self._generate_equality_func_helper(data_type, field, other_obj_name) self.emit('return YES;') self.emit() def _generate_equality_func_helper(self, data_type, field, other_obj_name): _, nullable = unwrap_nullable(field.data_type) if is_union_type(data_type): enum_field_name = fmt_enum_name(field.name, data_type) self.emit('case {}:'.format(enum_field_name)) if nullable: with self.block('if (self.{})'.format(fmt_var(field.name))): self._generate_equality_check(data_type, field, other_obj_name) else: self._generate_equality_check(data_type, field, other_obj_name) def _generate_equality_check(self, data_type, field, other_obj_name): field_name = fmt_var(field.name) if is_union_type(data_type): if is_void_type(field.data_type): self.emit('return [[self tagName] isEqual:[{} tagName]];'.format(other_obj_name)) else: self.emit('return [self.{0} isEqual:{1}.{0}];'.format( field_name, other_obj_name)) elif is_struct_type(data_type): with self.block('if (![self.{0} isEqual:{1}.{0}])'.format( field_name, other_obj_name)): self.emit('return NO;') def _cstor_args_from_fields(self, fields, is_struct=False): """Returns a string representing the properly formatted arguments for a constructor.""" if is_struct: args = [(fmt_var(f.name), fmt_type(f.data_type, tag=True, has_default=f.has_default)) for f in fields] else: args = [(fmt_var(f.name), fmt_type(f.data_type, tag=True)) for f in fields] return fmt_func_args_declaration(args) def _generate_validator(self, field): """Emits validator if data type has associated validator.""" validator = self._determine_validator_type(field.data_type, fmt_var(field.name), field.has_default) value = fmt_var( field.name) if not field.has_default else '{} ?: {}'.format( fmt_var(field.name), fmt_default_value(field)) if validator: self.emit('{}({});'.format(validator, value)) def _determine_validator_type(self, data_type, value, has_default): """Returns validator string for given data type, else None.""" data_type, nullable = unwrap_nullable(data_type) validator = None if is_list_type(data_type): item_validator = self._determine_validator_type( data_type.data_type, value, False) item_validator = item_validator if item_validator else 'nil' validator = '{}:{}'.format( fmt_validator(data_type), fmt_func_args([ ('minItems', '@({})'.format(data_type.min_items) if data_type.min_items else 'nil'), ('maxItems', '@({})'.format(data_type.max_items) if data_type.max_items else 'nil'), ('itemValidator', item_validator), ])) elif is_map_type(data_type): item_validator = self._determine_validator_type( data_type.value_data_type, value, False) item_validator = item_validator if item_validator else 'nil' validator = '{}:{}'.format( fmt_validator(data_type), fmt_func_args([ ('itemValidator', item_validator), ])) elif is_numeric_type(data_type): if data_type.min_value or data_type.max_value: validator = '{}:{}'.format( fmt_validator(data_type), fmt_func_args([ ('minValue', '@({})'.format(data_type.min_value) if data_type.min_value else 'nil'), ('maxValue', '@({})'.format(data_type.max_value) if data_type.max_value else 'nil'), ])) elif is_string_type(data_type): if data_type.pattern or data_type.min_length or data_type.max_length: pattern = data_type.pattern.encode('unicode_escape').replace( six.ensure_binary("\""), six.ensure_binary("\\\"")) if data_type.pattern else None validator = '{}:{}'.format( fmt_validator(data_type), fmt_func_args([ ('minLength', '@({})'.format(data_type.min_length) if data_type.min_length else 'nil'), ('maxLength', '@({})'.format(data_type.max_length) if data_type.max_length else 'nil'), ('pattern', '@"{}"'.format(six.ensure_str(pattern)) if pattern else 'nil'), ])) if nullable: if validator: validator = fmt_func_call( caller='DBStoneValidators', callee=validator) validator = fmt_func_call( caller='DBStoneValidators', callee='nullableValidator', args=validator) else: if validator: validator = fmt_func_call( caller='DBStoneValidators', callee=validator) else: validator = 'nil' if not has_default: validator = fmt_func_call( caller='DBStoneValidators', callee='nonnullValidator', args=validator) else: validator = None return validator def _generate_struct_serializer(self, struct): """Emits the serialize method for the serialization object for the given struct.""" struct_name = fmt_class_prefix(struct) with self.block_func( func='serialize', args=fmt_func_args_declaration([('valueObj', '{} *'.format(struct_name))]), return_type='NSDictionary *', class_func=True): if not struct.all_fields and not struct.has_enumerated_subtypes(): self.emit('#pragma unused(valueObj)') self.emit( 'NSMutableDictionary *jsonDict = [[NSMutableDictionary alloc] init];' ) self.emit() for field in struct.all_fields: data_type, nullable = unwrap_nullable(field.data_type) input_value = 'valueObj.{}'.format(fmt_var(field.name)) serialize_call = self._fmt_serialization_call( field.data_type, input_value, True) if not nullable: if is_primitive_type(data_type): self.emit('jsonDict[@"{}"] = {};'.format( field.name, input_value)) else: self.emit('jsonDict[@"{}"] = {};'.format( field.name, serialize_call)) else: with self.block('if ({})'.format(input_value)): self.emit('jsonDict[@"{}"] = {};'.format( field.name, serialize_call)) self.emit() if struct.has_enumerated_subtypes(): first_block = True for tags, subtype in struct.get_all_subtypes_with_tags(): assert len(tags) == 1, tags tag = tags[0] base_condition = '{} ([valueObj isKindOfClass:[{} class]])' with self.block( base_condition.format('if' if first_block else 'else if', fmt_class_prefix(subtype))): if first_block: first_block = False func_args = fmt_func_args([('value', '({} *)valueObj'.format( fmt_class_prefix( subtype)))]) caller = fmt_serial_class(fmt_class_prefix(subtype)) serialize_call = fmt_func_call( caller=caller, callee='serialize', args=func_args) self.emit('NSDictionary *subTypeFields = {};'.format( serialize_call)) with self.block( 'for (NSString* key in subTypeFields)'): self.emit('jsonDict[key] = subTypeFields[key];') self.emit( 'jsonDict[@".tag"] = @"{}";'.format(fmt_var(tag))) self.emit() self.emit('return [jsonDict count] > 0 ? jsonDict : nil;') self.emit() def _generate_struct_deserializer(self, struct): """Emits the deserialize method for the serialization object for the given struct.""" struct_name = fmt_class_prefix(struct) with self.block_func( func='deserialize', args=fmt_func_args_declaration([('valueDict', 'NSDictionary *')]), return_type='{} *'.format(struct_name), class_func=True): if not struct.all_fields and not struct.has_enumerated_subtypes(): self.emit('#pragma unused(valueDict)') def emit_struct_deserialize_logic(struct): for field in struct.all_fields: data_type, nullable = unwrap_nullable(field.data_type) input_value = 'valueDict[@"{}"]'.format(field.name) if is_primitive_type(data_type): deserialize_call = input_value else: deserialize_call = self._fmt_serialization_call( field.data_type, input_value, False) if nullable or field.has_default: default_value = fmt_default_value( field) if field.has_default else 'nil' if is_primitive_type(data_type): deserialize_call = '{} ?: {}'.format( input_value, default_value) else: deserialize_call = '{} ? {} : {}'.format( input_value, deserialize_call, default_value) self.emit('{}{} = {};'.format( fmt_type(field.data_type), fmt_var(field.name), deserialize_call)) self.emit() deserialized_obj_args = [(fmt_var(f.name), fmt_var(f.name)) for f in struct.all_fields] init_call = fmt_func_call( caller=fmt_alloc_call(caller=struct_name), callee=self._cstor_name_from_fields(struct.all_fields), args=fmt_func_args(deserialized_obj_args)) self.emit('return {};'.format(init_call)) if not struct.has_enumerated_subtypes(): emit_struct_deserialize_logic(struct) else: for tags, subtype in struct.get_all_subtypes_with_tags(): assert len(tags) == 1, tags tag = tags[0] base_string = 'if ([valueDict[@".tag"] isEqualToString:@"{}"])' with self.block(base_string.format(tag)): caller = fmt_serial_class(fmt_class_prefix(subtype)) args = fmt_func_args([('value', 'valueDict')]) deserialize_call = fmt_func_call( caller=caller, callee='deserialize', args=args) self.emit('return {};'.format(deserialize_call)) self.emit() if struct.is_catch_all(): emit_struct_deserialize_logic(struct) else: description_str = ( '[NSString stringWithFormat:@"Tag has an invalid ' 'value: \\\"%@\\\".", valueDict[@".tag"]]') self._generate_throw_error('InvalidTag', description_str) self.emit() def _generate_union_serializer(self, union): """Emits the serialize method for the serialization object for the given union.""" union_name = fmt_class_prefix(union) with self.block_func( func='serialize', args=fmt_func_args_declaration([('valueObj', '{} *'.format(union_name))]), return_type='NSDictionary *', class_func=True): if not union.all_fields: self.emit('#pragma unused(valueObj)') self.emit( 'NSMutableDictionary *jsonDict = [[NSMutableDictionary alloc] init];' ) self.emit() first_block = True for field in union.all_fields: with self.block('{} ([valueObj is{}])'.format( 'if' if first_block else 'else if', fmt_camel_upper(field.name))): data_type, nullable = unwrap_nullable(field.data_type) input_value = 'valueObj.{}'.format(fmt_var(field.name)) serialize_call = self._fmt_serialization_call( field.data_type, input_value, True) def emit_serializer(): if is_user_defined_type(data_type): if is_struct_type(data_type) and \ not data_type.has_enumerated_subtypes(): self.emit('[jsonDict addEntriesFromDictionary:{}];'. format(serialize_call)) else: self.emit( 'jsonDict[@"{}"] = [{} mutableCopy];'. format(field.name, serialize_call)) elif is_primitive_type(data_type): self.emit('jsonDict[@"{}"] = {};'.format( field.name, input_value)) else: self.emit('jsonDict[@"{}"] = {};'.format( field.name, serialize_call)) if not is_void_type(data_type): if not nullable: emit_serializer() else: with self.block('if (valueObj.{})'.format( fmt_var(field.name))): emit_serializer() self.emit('jsonDict[@".tag"] = @"{}";'.format(field.name)) if first_block: first_block = False with self.block('else'): if not union.closed: self.emit('jsonDict[@".tag"] = @"other";') else: self._generate_throw_error( 'InvalidTag', '@"Object not properly initialized. Tag has an unknown value."' ) self.emit() self.emit('return [jsonDict count] > 0 ? jsonDict : nil;') self.emit() def _generate_union_deserializer(self, union): """Emits the deserialize method for the serialization object for the given union.""" union_name = fmt_class_prefix(union) with self.block_func( func='deserialize', args=fmt_func_args_declaration([('valueDict', 'NSDictionary *')]), return_type='{} *'.format(union_name), class_func=True): if not union.all_fields: self.emit('#pragma unused(valueDict)') self.emit('NSString *tag = valueDict[@".tag"];') self.emit() first_block = True for field in union.all_fields: base_cond = '{} ([tag isEqualToString:@"{}"])' with self.block( base_cond.format('if' if first_block else 'else if', field.name)): if first_block: first_block = False if not is_void_type(field.data_type): data_type, nullable = unwrap_nullable(field.data_type) if is_struct_type( data_type ) and not data_type.has_enumerated_subtypes(): input_value = 'valueDict' else: input_value = 'valueDict[@"{}"]'.format(field.name) if is_primitive_type(data_type): deserialize_call = input_value else: deserialize_call = self._fmt_serialization_call( data_type, input_value, False) if nullable: deserialize_call = '{} ? {} : nil'.format( input_value, deserialize_call) self.emit('{}{} = {};'.format( fmt_type(field.data_type), fmt_var(field.name), deserialize_call)) deserialized_obj_args = [(fmt_var(field.name), fmt_var(field.name))] else: deserialized_obj_args = [] args = fmt_func_args(deserialized_obj_args) callee = self._cstor_name_from_field(field) self.emit('return {};'.format( fmt_func_call( caller=fmt_alloc_call(union_name), callee=callee, args=args))) with self.block('else'): if not union.closed: callee = 'initWithOther' self.emit('return {};'.format( fmt_func_call( caller=fmt_alloc_call(union_name), callee=callee))) else: reason = ( '[NSString stringWithFormat:@"Tag has an ' 'invalid value: \\\"%@\\\".", valueDict[@".tag"]]') self._generate_throw_error('InvalidTag', reason) self.emit() def _fmt_serialization_call(self, data_type, input_value, serialize, depth=0): """Returns the appropriate serialization / deserialization method call for the given data type.""" data_type, _ = unwrap_nullable(data_type) serializer_func = 'serialize' if serialize else 'deserialize' serializer_args = [] if is_primitive_type(data_type): return input_value if is_list_type(data_type) or is_map_type(data_type): serializer_args.append(('value', input_value)) elem_data_type = (data_type.value_data_type if is_map_type(data_type) else data_type.data_type) serialization_call = self._fmt_serialization_call( elem_data_type, 'elem{}'.format(depth), serialize, depth + 1) data_struct_block = '^id(id elem{}) {{ return {}; }}'.format( depth, serialization_call) serializer_args.append(('withBlock', data_struct_block)) elif is_timestamp_type(data_type): serializer_args.append(('value', input_value)) serializer_args.append(('dateFormat', '@"{}"'.format(data_type.format))) else: serializer_args.append(('value', input_value)) return '{}'.format( fmt_func_call( caller=fmt_serial_obj(data_type), callee=serializer_func, args=fmt_func_args(serializer_args))) def _generate_route_objects_m(self, route_schema, namespace): """Emits implementation files for Route objects which encapsulate information regarding each route. These objects are passed as parameters when route calls are made.""" output_path = 'Routes/RouteObjects/{}.m'.format( fmt_route_obj_class(namespace.name)) with self.output_to_relative_path(output_path): self.emit_raw(base_file_comment) import_classes = [ fmt_route_obj_class(namespace.name), 'DBStoneBase', 'DBRequestErrors', ] for auth_type in self.namespace_to_has_route_auth_list[namespace]: import_classes.append( fmt_routes_class(namespace.name, auth_type)) imports_classes_m = import_classes + \ self._get_imports_m( self._get_namespace_route_imports(namespace, include_route_args=False), []) self._generate_imports_m(imports_classes_m) with self.block_m(fmt_route_obj_class(namespace.name)): for route in namespace.routes: route_name = fmt_route_var(namespace.name, route) self.emit('static DBRoute *{};'.format(route_name)) self.emit() self.emit('static NSObject *lockObj = nil;') with self.block_func( func='initialize', args=[], return_type='void', class_func=True): self.emit('static dispatch_once_t onceToken;') with self.block( 'dispatch_once(&onceToken, ^{', delim=(None, None), after='});'): self.emit('lockObj = [[NSObject alloc] init];') self.emit() self.emit() for route in namespace.routes: route_name = fmt_route_var(namespace.name, route) if route.version == 1: route_path = route.name else: route_path = '{}_v{}'.format(route.name, route.version) if route.deprecated: deprecated = '@{}'.format('YES') else: deprecated = '@{}'.format('NO') if not is_void_type(route.result_data_type): caller = fmt_class_type( route.result_data_type, suppress_ptr=True) result_type = fmt_func_call( caller=caller, callee='class') else: result_type = 'nil' if not is_void_type(route.error_data_type): caller = fmt_class_type( route.error_data_type, suppress_ptr=True) error_type = fmt_func_call( caller=caller, callee='class') else: error_type = 'nil' if is_list_type(route.arg_data_type) or is_map_type(route.arg_data_type): dataStructSerialBlock = '^id(id dataStruct) {{ return {}; }}'.format( self._fmt_serialization_call( route.result_data_type, 'dataStruct', True)) else: dataStructSerialBlock = 'nil' if is_list_type(route.result_data_type) or is_map_type(route.result_data_type): dataStructDeserialBlock = '^id(id dataStruct) {{ return {}; }}'.format( self._fmt_serialization_call( route.result_data_type, 'dataStruct', False)) else: dataStructDeserialBlock = 'nil' with self.block_func( func=route_name, args=[], return_type='DBRoute *', class_func=True): with self.block('@synchronized(lockObj)'): with self.block('if (!{})'.format(route_name)): with self.block( '{} = [[DBRoute alloc] init:'.format( route_name), delim=(None, None), after='];'): self.emit('@\"{}\"'.format(route_path)) self.emit('namespace_:@\"{}\"'.format( namespace.name)) self.emit('deprecated:{}'.format(deprecated)) self.emit('resultType:{}'.format(result_type)) self.emit('errorType:{}'.format(error_type)) attrs = [] for field in route_schema.fields: attr_key = field.name attr_val = ("@\"{}\"".format(route.attrs .get(attr_key)) if route.attrs .get(attr_key) else 'nil') attrs.append('@\"{}\": {}'.format( attr_key, attr_val)) self.generate_multiline_list( attrs, delim=('attrs:@{', '}'), compact=True) self.emit('dataStructSerialBlock:{}'.format( dataStructSerialBlock)) self.emit('dataStructDeserialBlock:{}'.format( dataStructDeserialBlock)) self.emit('return {};'.format(route_name)) self.emit() self.emit() def _generate_route_objects_h( self, route_schema, # pylint: disable=unused-argument namespace): """Emits header files for Route objects which encapsulate information regarding each route. These objects are passed as parameters when route calls are made.""" output_path = 'Routes/RouteObjects/{}.h'.format( fmt_route_obj_class(namespace.name)) with self.output_to_relative_path(output_path): self.emit_raw(base_file_comment) self.emit('#import ') self.emit() self._generate_imports_h(['DBRoute']) self.emit() self.emit('NS_ASSUME_NONNULL_BEGIN') self.emit() self.emit(comment_prefix) description_str = ( 'Stone route objects for the {} namespace. Each route in ' 'the {} namespace has its own static object, which contains ' 'information about the route.') self.emit_wrapped_text( description_str.format( fmt_class(namespace.name), fmt_class(namespace.name)), prefix=comment_prefix) self.emit(comment_prefix) with self.block_h(fmt_route_obj_class(namespace.name)): for route in namespace.routes: route_name = fmt_route_var(namespace.name, route) route_obj_access_signature = fmt_signature( func=route_name, args=None, return_type='DBRoute *', class_func=True) base_str = 'Accessor method for the {} route object.' self.emit_wrapped_text( base_str.format(fmt_route_func(route)), prefix=comment_prefix) self.emit('{};'.format(route_obj_access_signature)) self.emit() self.emit() self.emit('NS_ASSUME_NONNULL_END') self.emit() def _generate_union_tag_access_signatures(self, union): """Emits the is methods and tagName method signatures for determining tag state and retrieving human-readable value of tag state, respectively.""" for field in union.all_fields: self.emit(comment_prefix) base_str = 'Retrieves whether the union\'s current tag state has value "{}".' self.emit_wrapped_text( base_str.format(field.name), prefix=comment_prefix) self.emit(comment_prefix) if not is_void_type(field.data_type): warning_str = ( '@note Call this method and ensure it returns true before ' 'accessing the `{}` property, otherwise a runtime exception ' 'will be thrown.') self.emit_wrapped_text( warning_str.format(fmt_var(field.name)), prefix=comment_prefix) self.emit(comment_prefix) base_str = '@return Whether the union\'s current tag state has value "{}".' self.emit_wrapped_text( base_str.format(field.name), prefix=comment_prefix) self.emit(comment_prefix) is_tag_signature = fmt_signature( func='is{}'.format(fmt_camel_upper(field.name)), args=[], return_type='BOOL') self.emit('{};'.format(is_tag_signature)) self.emit() get_tag_name_signature = fmt_signature( func='tagName', args=None, return_type='NSString *') self.emit(comment_prefix) self.emit_wrapped_text( "Retrieves string value of union's current tag state.", prefix=comment_prefix) self.emit(comment_prefix) base_str = "@return A human-readable string representing the union's current tag state." self.emit_wrapped_text(base_str, prefix=comment_prefix) self.emit(comment_prefix) self.emit('{};'.format(get_tag_name_signature)) self.emit() def _generate_union_tag_state_funcs(self, union): """Emits the is methods and tagName method for determining tag state and retrieving human-readable value of tag state, respectively.""" for field in union.all_fields: enum_field_name = fmt_enum_name(field.name, union) with self.block_func( func='is{}'.format(fmt_camel_upper(field.name)), args=[], return_type='BOOL'): self.emit('return _tag == {};'.format(enum_field_name)) self.emit() with self.block_func( func='tagName', args=[], return_type='NSString *'): with self.block('switch (_tag)'): for field in union.all_fields: enum_field_name = fmt_enum_name(field.name, union) self.emit('case {}:'.format(enum_field_name)) self.emit(' return @"{}";'.format(enum_field_name)) self.emit() self._generate_throw_error('InvalidTag', '@"Tag has an unknown value."') self.emit() def _generate_union_tag_vars_funcs(self, union): """Emits the getter methods for retrieving tag-specific state. Setters throw an error in the event an associated tag state variable is accessed without the correct tag state.""" for field in union.all_fields: if not is_void_type(field.data_type): enum_field_name = fmt_enum_name(field.name, union) with self.block_func( func=fmt_camel(field.name), args=[], return_type=fmt_type(field.data_type)): with self.block( 'if (![self is{}])'.format( fmt_camel_upper(field.name)), delim=('{', '}')): error_msg = 'Invalid tag: required {}, but was %@.'.format( enum_field_name) throw_exc = ( '[NSException raise:@"IllegalStateException" ' 'format:@"{}", [self tagName]];') self.emit(throw_exc.format(error_msg)) self.emit('return _{};'.format(fmt_var(field.name))) self.emit() def _generate_struct_properties(self, fields): """Emits struct instance properties from the given fields.""" for field in fields: doc = self.process_doc(field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text( self.process_doc(doc, self._docf), prefix=comment_prefix) self.emit(fmt_property(field=field)) self.emit() def _generate_union_properties(self, fields): """Emits union instance properties from the given fields.""" for field in fields: # void types do not need properties to store additional state # information if not is_void_type(field.data_type): doc = self.process_doc( field.doc, self._docf) if field.doc else undocumented warning_str = ( ' @note Ensure the `is{}` method returns true before accessing, ' 'otherwise a runtime exception will be raised.') doc += warning_str.format(fmt_camel_upper(field.name)) self.emit_wrapped_text( self.process_doc(doc, self._docf), prefix=comment_prefix) self.emit(fmt_property(field=field)) self.emit() def _generate_union_tag_property(self, union): """Emits union instance property representing union state.""" self.emit_wrapped_text( 'Represents the union\'s current tag state.', prefix=comment_prefix) self.emit( fmt_property_str( prop='tag', typ='{}'.format(fmt_enum_name('tag', union)))) self.emit() def _generate_class_comment(self, data_type): """Emits a generic class comment for a union or struct.""" if is_struct_type(data_type): class_type = 'struct' elif is_union_type(data_type): class_type = 'union' else: raise TypeError('Can\'t handle type %r' % type(data_type)) self.emit(comment_prefix) self.emit_wrapped_text( 'The `{}` {}.'.format(fmt_class(data_type.name), class_type), prefix=comment_prefix) if data_type.doc: self.emit(comment_prefix) self.emit_wrapped_text( self.process_doc(data_type.doc, self._docf), prefix=comment_prefix) self.emit(comment_prefix) protocol_str = ( 'This class implements the `DBSerializable` protocol ' '(serialize and deserialize instance methods), which is required ' 'for all Obj-C SDK API route objects.') self.emit_wrapped_text( protocol_str.format(fmt_class_prefix(data_type), class_type), prefix=comment_prefix) self.emit(comment_prefix) def _generate_throw_error(self, name, reason): """Emits a generic error throwing line.""" throw_exc = '@throw([NSException exceptionWithName:@"{}" reason:{} userInfo:nil]);' self.emit(throw_exc.format(name, reason)) def _docf(self, tag, val): if tag == 'route': return '`{}`'.format(fmt_func(val)) elif tag == 'field': if '.' in val: cls_name, field = val.split('.') return ('`{}` in `{}`'.format( fmt_var(field), self.obj_name_to_namespace[cls_name])) else: return fmt_var(val) elif tag in ('type', 'val', 'link'): return val else: return val stone-3.3.1/stone/backends/python_client.py000066400000000000000000000574351417406541500210260ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import re from stone.backend import CodeBackend from stone.backends.helpers import fmt_underscores from stone.backends.python_helpers import ( check_route_name_conflict, fmt_class, fmt_func, fmt_namespace, fmt_obj, fmt_type, fmt_var, ) from stone.backends.python_types import ( class_name_for_data_type, ) from stone.ir import ( is_nullable_type, is_list_type, is_map_type, is_struct_type, is_tag_ref, is_union_type, is_user_defined_type, is_void_type, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any # This will be at the top of the generated file. base = """\ # -*- coding: utf-8 -*- # Auto-generated by Stone, do not modify. # flake8: noqa # pylint: skip-file from abc import ABCMeta, abstractmethod """ # Matches format of Babel doc tags doc_sub_tag_re = re.compile(':(?P[A-z]*):`(?P.*?)`') DOCSTRING_CLOSE_RESPONSE = """\ If you do not consume the entire response body, then you must call close on the response object, otherwise you will max out your available connections. We recommend using the `contextlib.closing `_ context manager to ensure this.""" _cmdline_parser = argparse.ArgumentParser( prog='python-client-backend', description=( 'Generates a Python class with a method for each route. Extend the ' 'generated class and implement the abstract request() method. This ' 'class assumes that the python_types backend was used with the same ' 'output directory.'), ) _cmdline_parser.add_argument( '-m', '--module-name', required=True, type=str, help=('The name of the Python module to generate. Please exclude the .py ' 'file extension.'), ) _cmdline_parser.add_argument( '-c', '--class-name', required=True, type=str, help='The name of the Python class that contains each route as a method.', ) _cmdline_parser.add_argument( '-t', '--types-package', required=True, type=str, help='The output Python package of the python_types backend.', ) _cmdline_parser.add_argument( '-e', '--error-class-path', default='.exceptions.ApiError', type=str, help=( "The path to the class that's raised when a route returns an error. " "The class name is inserted into the doc for route methods."), ) _cmdline_parser.add_argument( '-w', '--auth-type', type=str, help='The auth type of the client to generate.', ) _cmdline_parser.add_argument( '-a', '--attribute-comment', action='append', type=str, default=[], help=('Route attributes that the backend will have access to and ' 'presumably expose in generated code. Use ":all" to select all ' 'attributes defined in stone_cfg.Route. Attributes will be ' "exposed in the documentation, as the client doesn't use them."), ) class PythonClientBackend(CodeBackend): cmdline_parser = _cmdline_parser supported_auth_types = None def generate(self, api): """Generates a module called "base". The module will contain a base class that will have a method for each route across all namespaces. """ with self.output_to_relative_path('%s.py' % self.args.module_name): self.emit_raw(base) # Import "warnings" if any of the routes are deprecated. found_deprecated = False for namespace in api.namespaces.values(): for route in namespace.routes: if route.deprecated: self.emit('import warnings') found_deprecated = True break if found_deprecated: break self.emit() self._generate_imports(api.namespaces.values()) self.emit() self.emit() # PEP-8 expects two-blank lines before class def self.emit('class %s(object):' % self.args.class_name) with self.indent(): self.emit('__metaclass__ = ABCMeta') self.emit() self.emit('@abstractmethod') self.emit( 'def request(self, route, namespace, arg, arg_binary=None):') with self.indent(): self.emit('pass') self.emit() self._generate_route_methods(api.namespaces.values()) def _generate_imports(self, namespaces): # Only import namespaces that have user-defined types defined. for namespace in namespaces: if namespace.data_types: self.emit('from {} import {}'.format(self.args.types_package, fmt_namespace(namespace.name))) def _generate_route_methods(self, namespaces): """Creates methods for the routes in each namespace. All data types and routes are represented as Python classes.""" self.cur_namespace = None for namespace in namespaces: if namespace.routes: self.emit('# ------------------------------------------') self.emit('# Routes in {} namespace'.format(namespace.name)) self.emit() self._generate_routes(namespace) def _generate_routes(self, namespace): """ Generates Python methods that correspond to routes in the namespace. """ # Hack: needed for _docf() self.cur_namespace = namespace # list of auth_types supported in this base class. # this is passed with the new -w flag if self.args.auth_type is not None: self.supported_auth_types = [auth_type.strip().lower() for auth_type in self.args.auth_type.split(',')] check_route_name_conflict(namespace) for route in namespace.routes: # compatibility mode : included routes are passed by whitelist # actual auth attr inluded in the route is ignored in this mode. if self.supported_auth_types is None: self._generate_route_helper(namespace, route) if route.attrs.get('style') == 'download': self._generate_route_helper(namespace, route, True) else: route_auth_attr = None if route.attrs is not None: route_auth_attr = route.attrs.get('auth') if route_auth_attr is None: continue route_auth_modes = [mode.strip().lower() for mode in route_auth_attr.split(',')] for base_auth_type in self.supported_auth_types: if base_auth_type in route_auth_modes: self._generate_route_helper(namespace, route) if route.attrs.get('style') == 'download': self._generate_route_helper(namespace, route, True) break # to avoid duplicate method declaration in the same base class def _generate_route_helper(self, namespace, route, download_to_file=False): """Generate a Python method that corresponds to a route. :param namespace: Namespace that the route belongs to. :param stone.ir.ApiRoute route: IR node for the route. :param bool download_to_file: Whether a special version of the route that downloads the response body to a file should be generated. This can only be used for download-style routes. """ arg_data_type = route.arg_data_type result_data_type = route.result_data_type request_binary_body = route.attrs.get('style') == 'upload' response_binary_body = route.attrs.get('style') == 'download' if download_to_file: assert response_binary_body, 'download_to_file can only be set ' \ 'for download-style routes.' self._generate_route_method_decl(namespace, route, arg_data_type, request_binary_body, method_name_suffix='_to_file', extra_args=['download_path']) else: self._generate_route_method_decl(namespace, route, arg_data_type, request_binary_body) with self.indent(): extra_request_args = None extra_return_arg = None footer = None if request_binary_body: extra_request_args = [('f', 'bytes', 'Contents to upload.')] elif download_to_file: extra_request_args = [('download_path', 'str', 'Path on local machine to save file.')] if response_binary_body and not download_to_file: extra_return_arg = ':class:`requests.models.Response`' footer = DOCSTRING_CLOSE_RESPONSE if route.doc: func_docstring = self.process_doc(route.doc, self._docf) else: func_docstring = None self._generate_docstring_for_func( namespace, arg_data_type, result_data_type, route.error_data_type, overview=func_docstring, extra_request_args=extra_request_args, extra_return_arg=extra_return_arg, footer=footer, attrs=route.attrs, ) self._maybe_generate_deprecation_warning(route) # Code to instantiate a class for the request data type if is_void_type(arg_data_type): self.emit('arg = None') elif is_struct_type(arg_data_type): self.generate_multiline_list( [f.name for f in arg_data_type.all_fields], before='arg = {}.{}'.format( fmt_namespace(arg_data_type.namespace.name), fmt_class(arg_data_type.name)), ) elif not is_union_type(arg_data_type): raise AssertionError('Unhandled request type %r' % arg_data_type) # Code to make the request args = [ '{}.{}'.format(fmt_namespace(namespace.name), fmt_func(route.name, version=route.version)), "'{}'".format(namespace.name), 'arg'] if request_binary_body: args.append('f') else: args.append('None') self.generate_multiline_list(args, 'r = self.request', compact=False) if download_to_file: self.emit('self._save_body_to_file(download_path, r[1])') if is_void_type(result_data_type): self.emit('return None') else: self.emit('return r[0]') else: if is_void_type(result_data_type): self.emit('return None') else: self.emit('return r') self.emit() def _generate_route_method_decl( self, namespace, route, arg_data_type, request_binary_body, method_name_suffix='', extra_args=None): """Generates the method prototype for a route.""" args = ['self'] if extra_args: args += extra_args if request_binary_body: args.append('f') if is_struct_type(arg_data_type): for field in arg_data_type.all_fields: if is_nullable_type(field.data_type): args.append('{}=None'.format(field.name)) elif field.has_default: # TODO(kelkabany): Decide whether we really want to set the # default in the argument list. This will send the default # over the wire even if it isn't overridden. The benefit is # it locks in a default even if it is changed server-side. if is_user_defined_type(field.data_type): ns = field.data_type.namespace else: ns = None arg = '{}={}'.format( field.name, self._generate_python_value(ns, field.default)) args.append(arg) else: args.append(field.name) elif is_union_type(arg_data_type): args.append('arg') elif not is_void_type(arg_data_type): raise AssertionError('Unhandled request type: %r' % arg_data_type) method_name = fmt_func(route.name + method_name_suffix, version=route.version) namespace_name = fmt_underscores(namespace.name) self.generate_multiline_list(args, 'def {}_{}'.format(namespace_name, method_name), ':') def _maybe_generate_deprecation_warning(self, route): if route.deprecated: msg = '{} is deprecated.'.format(route.name) if route.deprecated.by: msg += ' Use {}.'.format(route.deprecated.by.name) args = ["'{}'".format(msg), 'DeprecationWarning'] self.generate_multiline_list( args, before='warnings.warn', delim=('(', ')'), compact=False, ) def _generate_docstring_for_func(self, namespace, arg_data_type, result_data_type=None, error_data_type=None, overview=None, extra_request_args=None, extra_return_arg=None, footer=None, attrs=None): """ Generates a docstring for a function or method. This function is versatile. It will create a docstring using all the data that is provided. :param arg_data_type: The data type describing the argument to the route. The data type should be a struct, and each field will be treated as an input parameter of the method. :param result_data_type: The data type of the route result. :param error_data_type: The data type of the route result in the case of an error. :param str overview: A description of the route that will be located at the top of the docstring. :param extra_request_args: [(field name, field type, field doc), ...] Describes any additional parameters for the method that aren't a field in arg_data_type. :param str extra_return_arg: Name of an additional return type that. If this is specified, it is assumed that the return of the function will be a tuple of return_data_type and extra_return-arg. :param str footer: Additional notes at the end of the docstring. """ fields = [] if is_void_type(arg_data_type) else arg_data_type.fields attrs_lines = [] if self.args.attribute_comment and attrs: for attribute in self.args.attribute_comment: if attribute in attrs and attrs[attribute] is not None: attrs_lines.append('{}: {}'.format(attribute, attrs[attribute])) if not fields and not overview and not attrs_lines: # If we don't have an overview or any input parameters, we skip the # docstring altogether. return self.emit('"""') if overview: self.emit_wrapped_text(overview) if attrs_lines: if overview: self.emit() self.emit('Route attributes:') for a in attrs_lines: self.emit_wrapped_text(a, ' ') # Description of all input parameters if extra_request_args or fields: if overview or attrs_lines: # Add a blank line if we had an overview or attrs self.emit() if extra_request_args: for name, data_type_name, doc in extra_request_args: if data_type_name: field_doc = ':param {} {}: {}'.format(data_type_name, name, doc) self.emit_wrapped_text(field_doc, subsequent_prefix=' ') else: self.emit_wrapped_text( ':param {}: {}'.format(name, doc), subsequent_prefix=' ') if is_struct_type(arg_data_type): for field in fields: if field.doc: if is_user_defined_type(field.data_type): field_doc = ':param {}: {}'.format( field.name, self.process_doc(field.doc, self._docf)) else: field_doc = ':param {} {}: {}'.format( self._format_type_in_doc(namespace, field.data_type), field.name, self.process_doc(field.doc, self._docf), ) self.emit_wrapped_text( field_doc, subsequent_prefix=' ') if is_user_defined_type(field.data_type): # It's clearer to declare the type of a composite on # a separate line since it references a class in # another module self.emit(':type {}: {}'.format( field.name, self._format_type_in_doc(namespace, field.data_type), )) else: # If the field has no docstring, then just document its # type. field_doc = ':type {}: {}'.format( field.name, self._format_type_in_doc(namespace, field.data_type), ) self.emit_wrapped_text(field_doc) elif is_union_type(arg_data_type): if arg_data_type.doc: self.emit_wrapped_text(':param arg: {}'.format( self.process_doc(arg_data_type.doc, self._docf)), subsequent_prefix=' ') self.emit(':type arg: {}'.format( self._format_type_in_doc(namespace, arg_data_type))) if (overview or attrs_lines) and not (extra_request_args or fields): # Only output an empty line if we had an overview and haven't # started a section on declaring types. self.emit() if extra_return_arg: # Special case where the function returns a tuple. The first # element is the JSON response. The second element is the # the extra_return_arg param. args = [] if is_void_type(result_data_type): args.append('None') else: rtype = self._format_type_in_doc(namespace, result_data_type) args.append(rtype) args.append(extra_return_arg) self.generate_multiline_list(args, ':rtype: ') else: if is_void_type(result_data_type): self.emit(':rtype: None') else: rtype = self._format_type_in_doc(namespace, result_data_type) self.emit(':rtype: {}'.format(rtype)) if not is_void_type(error_data_type) and error_data_type.fields: self.emit(':raises: :class:`{}`'.format(self.args.error_class_path)) self.emit() # To provide more clarity to a dev who reads the docstring, suggest # the route's error class. This is confusing, however, because we # don't know where the error object that's raised will store # the more detailed route error defined in stone. error_class_name = self.args.error_class_path.rsplit('.', 1)[-1] self.emit('If this raises, {} will contain:'.format(error_class_name)) with self.indent(): self.emit(self._format_type_in_doc(namespace, error_data_type)) if footer: self.emit() self.emit_wrapped_text(footer) self.emit('"""') def _docf(self, tag, val): """ Callback used as the handler argument to process_docs(). This converts Babel doc references to Sphinx-friendly annotations. """ if tag == 'type': fq_val = val if '.' not in val: fq_val = self.cur_namespace.name + '.' + fq_val return ':class:`{}.{}`'.format(self.args.types_package, fq_val) elif tag == 'route': if ':' in val: val, version = val.split(':', 1) version = int(version) else: version = 1 if '.' in val: return ':meth:`{}`'.format(fmt_func(val, version=version)) else: return ':meth:`{}_{}`'.format( self.cur_namespace.name, fmt_func(val, version=version)) elif tag == 'link': anchor, link = val.rsplit(' ', 1) return '`{} <{}>`_'.format(anchor, link) elif tag == 'val': if val == 'null': return 'None' elif val == 'true' or val == 'false': return '``{}``'.format(val.capitalize()) else: return val elif tag == 'field': return '``{}``'.format(val) else: raise RuntimeError('Unknown doc ref tag %r' % tag) def _format_type_in_doc(self, namespace, data_type): """ Returns a string that can be recognized by Sphinx as a type reference in a docstring. """ if is_void_type(data_type): return 'None' elif is_user_defined_type(data_type): return ':class:`{}.{}.{}`'.format( self.args.types_package, namespace.name, fmt_type(data_type)) elif is_nullable_type(data_type): return 'Nullable[{}]'.format( self._format_type_in_doc(namespace, data_type.data_type), ) elif is_list_type(data_type): return 'List[{}]'.format( self._format_type_in_doc(namespace, data_type.data_type), ) elif is_map_type(data_type): return 'Map[{}, {}]'.format( self._format_type_in_doc(namespace, data_type.key_data_type), self._format_type_in_doc(namespace, data_type.value_data_type), ) else: return fmt_type(data_type) def _generate_python_value(self, namespace, value): if is_tag_ref(value): return '{}.{}.{}'.format( fmt_namespace(namespace.name), class_name_for_data_type(value.union_data_type), fmt_var(value.tag_name)) else: return fmt_obj(value) stone-3.3.1/stone/backends/python_helpers.py000066400000000000000000000136651417406541500212070ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from contextlib import contextmanager import pprint from stone.backend import Backend, CodeBackend from stone.backends.helpers import ( fmt_pascal, fmt_underscores, ) from stone.ir import ApiNamespace from stone.ir import ( AnnotationType, Boolean, Bytes, Float32, Float64, Int32, Int64, List, String, Timestamp, UInt32, UInt64, is_user_defined_type, is_alias, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression _type_table = { Boolean: 'bool', Bytes: 'bytes', Float32: 'float', Float64: 'float', Int32: 'int', Int64: 'int', List: 'list', String: 'str', Timestamp: 'datetime', UInt32: 'int', UInt64: 'int', } _reserved_keywords = { 'break', 'class', 'continue', 'for', 'pass', 'while', 'async', } @contextmanager def emit_pass_if_nothing_emitted(codegen): # type: (CodeBackend) -> typing.Iterator[None] starting_lineno = codegen.lineno yield ending_lineno = codegen.lineno if starting_lineno == ending_lineno: codegen.emit("pass") codegen.emit() def _rename_if_reserved(s): if s in _reserved_keywords: return s + '_' else: return s def fmt_class(name, check_reserved=False): s = fmt_pascal(name) return _rename_if_reserved(s) if check_reserved else s def fmt_func(name, check_reserved=False, version=1): name = fmt_underscores(name) if check_reserved: name = _rename_if_reserved(name) if version > 1: name = '{}_v{}'.format(name, version) return name def fmt_obj(o): return pprint.pformat(o, width=1) def fmt_type(data_type): return _type_table.get(data_type.__class__, fmt_class(data_type.name)) def fmt_var(name, check_reserved=False): s = fmt_underscores(name) return _rename_if_reserved(s) if check_reserved else s def fmt_namespaced_var(ns_name, data_type_name, field_name): return ".".join([ns_name, data_type_name, fmt_var(field_name)]) def fmt_namespace(name): return _rename_if_reserved(name) def check_route_name_conflict(namespace): """ Check name conflicts among generated route definitions. Raise a runtime exception when a conflict is encountered. """ route_by_name = {} for route in namespace.routes: route_name = fmt_func(route.name, version=route.version) if route_name in route_by_name: other_route = route_by_name[route_name] raise RuntimeError( 'There is a name conflict between {!r} and {!r}'.format(other_route, route)) route_by_name[route_name] = route TYPE_IGNORE_COMMENT = " # type: ignore" def generate_imports_for_referenced_namespaces( backend, namespace, package, insert_type_ignore=False): # type: (Backend, ApiNamespace, typing.Text, bool) -> None """ Both the true Python backend and the Python PEP 484 Type Stub backend have to perform the same imports. :param insert_type_ignore: add a MyPy type-ignore comment to the imports in the except: clause. """ imported_namespaces = namespace.get_imported_namespaces(consider_annotation_types=True) if not imported_namespaces: return type_ignore_comment = TYPE_IGNORE_COMMENT if insert_type_ignore else "" for ns in imported_namespaces: backend.emit('from {package} import {namespace_name}{type_ignore_comment}'.format( package=package, namespace_name=fmt_namespace(ns.name), type_ignore_comment=type_ignore_comment )) backend.emit() def generate_module_header(backend): backend.emit('# -*- coding: utf-8 -*-') backend.emit('# Auto-generated by Stone, do not modify.') # Silly way to not type ATgenerated in our code to avoid having this # file marked as auto-generated by our code review tool. backend.emit('# @{}'.format('generated')) backend.emit('# flake8: noqa') backend.emit('# pylint: skip-file') # This will be at the top of every generated file. _validators_import_template = """\ from stone.backends.python_rsrc import stone_base as bb{type_ignore_comment} from stone.backends.python_rsrc import stone_validators as bv{type_ignore_comment} """ validators_import = _validators_import_template.format(type_ignore_comment="") validators_import_with_type_ignore = _validators_import_template.format( type_ignore_comment=TYPE_IGNORE_COMMENT ) def prefix_with_ns_if_necessary(name, name_ns, source_ns): # type: (typing.Text, ApiNamespace, ApiNamespace) -> typing.Text """ Returns a name that can be used to reference `name` in namespace `name_ns` from `source_ns`. If `source_ns` and `name_ns` are the same, that's just `name`. Otherwise it's `name_ns`.`name`. """ if source_ns == name_ns: return name return '{}.{}'.format(fmt_namespace(name_ns.name), name) def class_name_for_data_type(data_type, ns=None): """ Returns the name of the Python class that maps to a user-defined type. The name is identical to the name in the spec. If ``ns`` is set to a Namespace and the namespace of `data_type` does not match, then a namespace prefix is added to the returned name. For example, ``foreign_ns.TypeName``. """ assert is_user_defined_type(data_type) or is_alias(data_type), \ 'Expected composite type, got %r' % type(data_type) name = fmt_class(data_type.name) if ns: return prefix_with_ns_if_necessary(name, data_type.namespace, ns) return name def class_name_for_annotation_type(annotation_type, ns=None): """ Same as class_name_for_data_type, but works with annotation types. """ assert isinstance(annotation_type, AnnotationType) name = fmt_class(annotation_type.name) if ns: return prefix_with_ns_if_necessary(name, annotation_type.namespace, ns) return name stone-3.3.1/stone/backends/python_rsrc/000077500000000000000000000000001417406541500201315ustar00rootroot00000000000000stone-3.3.1/stone/backends/python_rsrc/__init__.py000066400000000000000000000001111417406541500222330ustar00rootroot00000000000000# Make this a package so that the Python backend tests can import these. stone-3.3.1/stone/backends/python_rsrc/stone_base.py000066400000000000000000000205401417406541500226260ustar00rootroot00000000000000""" Helpers for representing Stone data types in Python. """ from __future__ import absolute_import, unicode_literals import functools from stone.backends.python_rsrc import stone_validators as bv _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class AnnotationType(object): # This is a base class for all annotation types. pass if _MYPY: T = typing.TypeVar('T', bound=AnnotationType) U = typing.TypeVar('U') class NotSet(object): __slots__ = () def __copy__(self): # type: () -> NotSet # disable copying so we can do identity comparison even after copying stone objects return self def __deepcopy__(self, memo): # type: (typing.Dict[typing.Text, typing.Any]) -> NotSet # disable copying so we can do identity comparison even after copying stone objects return self def __repr__(self): return "NOT_SET" NOT_SET = NotSet() # dummy object to denote that a field has not been set NO_DEFAULT = object() class Attribute(object): __slots__ = ("name", "default", "nullable", "user_defined", "validator") def __init__(self, name, nullable=False, user_defined=False): # type: (typing.Text, bool, bool) -> None # Internal name to store actual value for attribute. self.name = "_{}_value".format(name) self.nullable = nullable self.user_defined = user_defined # These should be set later, because of possible cross-references. self.validator = None # type: typing.Any self.default = NO_DEFAULT def __get__(self, instance, owner): # type: (typing.Any, typing.Any) -> typing.Any if instance is None: return self value = getattr(instance, self.name) if value is not NOT_SET: return value if self.nullable: return None if self.default is not NO_DEFAULT: return self.default # No luck, give a nice error. raise AttributeError("missing required field '{}'".format(public_name(self.name))) def __set__(self, instance, value): # type: (typing.Any, typing.Any) -> None if self.nullable and value is None: setattr(instance, self.name, NOT_SET) return if self.user_defined: self.validator.validate_type_only(value) else: value = self.validator.validate(value) setattr(instance, self.name, value) def __delete__(self, instance): # type: (typing.Any) -> None setattr(instance, self.name, NOT_SET) class Struct(object): # This is a base class for all classes representing Stone structs. # every parent class in the inheritance tree must define __slots__ in order to get full memory # savings __slots__ = () _all_field_names_ = set() # type: typing.Set[str] def __eq__(self, other): # type: (object) -> bool if not isinstance(other, Struct): return False if self._all_field_names_ != other._all_field_names_: return False if not isinstance(other, self.__class__) and not isinstance(self, other.__class__): return False for field_name in self._all_field_names_: if getattr(self, field_name) != getattr(other, field_name): return False return True def __ne__(self, other): # type: (object) -> bool return not self == other def __repr__(self): args = ["{}={!r}".format(name, getattr(self, "_{}_value".format(name))) for name in sorted(self._all_field_names_)] return "{}({})".format(type(self).__name__, ", ".join(args)) def _process_custom_annotations(self, annotation_type, field_path, processor): # type: (typing.Type[T], typing.Text, typing.Callable[[T, U], U]) -> None pass class Union(object): # TODO(kelkabany): Possible optimization is to remove _value if a # union is composed of only symbols. __slots__ = ['_tag', '_value'] _tagmap = {} # type: typing.Dict[str, bv.Validator] _permissioned_tagmaps = set() # type: typing.Set[typing.Text] def __init__(self, tag, value=None): validator = None tagmap_names = ['_{}_tagmap'.format(map_name) for map_name in self._permissioned_tagmaps] for tagmap_name in ['_tagmap'] + tagmap_names: if tag in getattr(self, tagmap_name): validator = getattr(self, tagmap_name)[tag] assert validator is not None, 'Invalid tag %r.' % tag if isinstance(validator, bv.Void): assert value is None, 'Void type union member must have None value.' elif isinstance(validator, (bv.Struct, bv.Union)): validator.validate_type_only(value) else: validator.validate(value) self._tag = tag self._value = value def __eq__(self, other): # Also need to check if one class is a subclass of another. If one union extends another, # the common fields should be able to be compared to each other. return ( isinstance(other, Union) and (isinstance(self, other.__class__) or isinstance(other, self.__class__)) and self._tag == other._tag and self._value == other._value ) def __ne__(self, other): return not self == other def __hash__(self): return hash((self._tag, self._value)) def __repr__(self): return "{}({!r}, {!r})".format(type(self).__name__, self._tag, self._value) def _process_custom_annotations(self, annotation_type, field_path, processor): # type: (typing.Type[T], typing.Text, typing.Callable[[T, U], U]) -> None pass @classmethod def _is_tag_present(cls, tag, caller_permissions): assert tag is not None, 'tag value should not be None' if tag in cls._tagmap: return True for extra_permission in caller_permissions.permissions: tagmap_name = '_{}_tagmap'.format(extra_permission) if hasattr(cls, tagmap_name) and tag in getattr(cls, tagmap_name): return True return False @classmethod def _get_val_data_type(cls, tag, caller_permissions): assert tag is not None, 'tag value should not be None' for extra_permission in caller_permissions.permissions: tagmap_name = '_{}_tagmap'.format(extra_permission) if hasattr(cls, tagmap_name) and tag in getattr(cls, tagmap_name): return getattr(cls, tagmap_name)[tag] return cls._tagmap[tag] class Route(object): __slots__ = ("name", "version", "deprecated", "arg_type", "result_type", "error_type", "attrs") def __init__(self, name, version, deprecated, arg_type, result_type, error_type, attrs): self.name = name self.version = version self.deprecated = deprecated self.arg_type = arg_type self.result_type = result_type self.error_type = error_type assert isinstance(attrs, dict), 'Expected dict, got %r' % attrs self.attrs = attrs def __repr__(self): return 'Route({!r}, {!r}, {!r}, {!r}, {!r}, {!r}, {!r})'.format( self.name, self.version, self.deprecated, self.arg_type, self.result_type, self.error_type, self.attrs) # helper functions used when constructing custom annotation processors # put this here so that every other file doesn't need to import functools partially_apply = functools.partial def make_struct_annotation_processor(annotation_type, processor): def g(field_path, struct): if struct is None: return struct struct._process_custom_annotations(annotation_type, field_path, processor) return struct return g def make_list_annotation_processor(processor): def g(field_path, list_): if list_ is None: return list_ return [processor('{}[{}]'.format(field_path, idx), x) for idx, x in enumerate(list_)] return g def make_map_value_annotation_processor(processor): def g(field_path, map_): if map_ is None: return map_ return {k: processor('{}[{}]'.format(field_path, repr(k)), v) for k, v in map_.items()} return g def public_name(name): # _some_attr_value -> some_attr return "_".join(name.split("_")[1:-1]) stone-3.3.1/stone/backends/python_rsrc/stone_serializers.py000066400000000000000000001235061417406541500242560ustar00rootroot00000000000000""" Serializers for Stone data types. Currently, only JSON is officially supported, but there's an experimental msgpack integration. If possible, serializers should be kept separate from the RPC format. This module should be dropped into a project that requires the use of Stone. In the future, this could be imported from a pre-installed Python package, rather than being added to a project. """ from __future__ import absolute_import, unicode_literals import base64 import binascii import collections import datetime import functools import json import re import time import six from stone.backends.python_rsrc import ( stone_base as bb, stone_validators as bv, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # ------------------------------------------------------------------------ class CallerPermissionsInterface(object): @property def permissions(self): """ Returns the list of permissions this caller has access to. """ raise NotImplementedError class CallerPermissionsDefault(CallerPermissionsInterface): @property def permissions(self): return [] # ------------------------------------------------------------------------ class StoneEncoderInterface(object): """ Interface defining a stone object encoder. """ def encode(self, validator, value): # type: (bv.Validator, typing.Any) -> typing.Any """ Validate ``value`` using ``validator`` and return the encoding. Args: validator: the ``stone_validators.Validator`` used to validate ``value`` value: the object to encode Returns: The encoded object. This is implementation-defined. Raises: stone_validators.ValidationError: Raised if ``value`` (or one of its sub-values). """ raise NotImplementedError # ------------------------------------------------------------------------ class StoneSerializerBase(StoneEncoderInterface): def __init__(self, caller_permissions, alias_validators): # type: (CallerPermissionsInterface, typing.Mapping[bv.Validator, typing.Callable[[typing.Any], None]]) -> None # noqa: E501 """ Constructor, `obviously `. Args: caller_permissions (list): The list of raw-string caller permissions with which to serialize. alias_validators (``typing.Mapping``, optional): A mapping of custom validation callables in the format ``{stone_validators.Validator: typing.Callable[[typing.Any], None], ...}``. These callables must raise a ``stone_validators.ValidationError`` on failure. Defaults to ``None``. """ self.caller_permissions = (caller_permissions if caller_permissions else CallerPermissionsDefault()) self._alias_validators = {} # type: typing.Dict[bv.Validator, typing.Callable[[typing.Any], None]] # noqa: E501 if alias_validators is not None: self._alias_validators.update(alias_validators) @property def alias_validators(self): """ A ``typing.Mapping`` of custom validation callables in the format ``{stone_validators.Validator: typing.Callable[typing.Any], ...}``. """ return self._alias_validators def encode(self, validator, value): return self.encode_sub(validator, value) def encode_sub(self, validator, value): # type: (bv.Validator, typing.Any) -> typing.Any """ Callback intended to be called by other ``encode`` methods to delegate encoding of sub-values. Arguments have the same semantics as with the ``encode`` method. """ if isinstance(validator, bv.List): # Because Lists are mutable, we always validate them during # serialization validate_f = validator.validate # type: typing.Callable[[typing.Any], None] encode_f = self.encode_list # type: typing.Callable[[typing.Any, typing.Any], typing.Any] # noqa: E501 elif isinstance(validator, bv.Map): # Also validate maps during serialization because they are also mutable validate_f = validator.validate encode_f = self.encode_map elif isinstance(validator, bv.Nullable): validate_f = validator.validate encode_f = self.encode_nullable elif isinstance(validator, bv.Primitive): validate_f = validator.validate encode_f = self.encode_primitive elif isinstance(validator, bv.Struct): if isinstance(validator, bv.StructTree): if self.caller_permissions.permissions: def validate_with_permissions(val): validator.validate_with_permissions(val, self.caller_permissions) validate_f = validate_with_permissions else: validate_f = validator.validate encode_f = self.encode_struct_tree else: # Fields are already validated on assignment if self.caller_permissions.permissions: def validate_with_permissions(val): validator.validate_with_permissions(val, self.caller_permissions) validate_f = validate_with_permissions else: validate_f = validator.validate_type_only encode_f = self.encode_struct elif isinstance(validator, bv.Union): # Fields are already validated on assignment validate_f = validator.validate_type_only encode_f = self.encode_union else: raise bv.ValidationError('Unsupported data type {}'.format(type(validator).__name__)) validate_f(value) return encode_f(validator, value) def encode_list(self, validator, value): # type: (bv.List, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.List``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_map(self, validator, value): # type: (bv.Map, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.Map``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_nullable(self, validator, value): # type: (bv.Nullable, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.Nullable``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_primitive(self, validator, value): # type: (bv.Primitive, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.Primitive``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_struct(self, validator, value): # type: (bv.Struct, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.Struct``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_struct_tree(self, validator, value): # type: (bv.StructTree, typing.Any) -> typing.Any """ Callback for serializing a ``stone_validators.StructTree``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError def encode_union(self, validator, value): # type: (bv.Union, bb.Union) -> typing.Any """ Callback for serializing a ``stone_validators.Union``. Arguments have the same semantics as with the ``encode`` method. """ raise NotImplementedError # ------------------------------------------------------------------------ class StoneToPythonPrimitiveSerializer(StoneSerializerBase): def __init__(self, caller_permissions, alias_validators, for_msgpack, old_style, should_redact): # type: (CallerPermissionsInterface, typing.Mapping[bv.Validator, typing.Callable[[typing.Any], None]], bool, bool, bool) -> None # noqa: E501 """ Args: alias_validators (``typing.Mapping``, optional): Passed to ``StoneSerializer.__init__``. Defaults to ``None``. for_msgpack (bool, optional): See the like-named property. Defaults to ``False``. old_style (bool, optional): See the like-named property. Defaults to ``False``. should_redact (bool, optional): Whether to perform redaction on marked fields. Defaults to ``False``. """ super(StoneToPythonPrimitiveSerializer, self).__init__( caller_permissions, alias_validators=alias_validators) self._for_msgpack = for_msgpack self._old_style = old_style self.should_redact = should_redact @property def for_msgpack(self): """ EXPERIMENTAL: A flag associated with the serializer indicating whether objects produced by the ``encode`` method should be encoded for msgpack. """ return self._for_msgpack @property def old_style(self): """ A flag associated with the serializer indicating whether objects produced by the ``encode`` method should be encoded according to Dropbox's old or new API styles. """ return self._old_style def encode_sub(self, validator, value): if self.should_redact and hasattr(validator, '_redact'): if isinstance(value, list): return [validator._redact.apply(v) for v in value] elif isinstance(value, dict): return {k: validator._redact.apply(v) for k, v in value.items()} else: return validator._redact.apply(value) # Encode value normally return super(StoneToPythonPrimitiveSerializer, self).encode_sub(validator, value) def encode_list(self, validator, value): validated_value = validator.validate(value) return [self.encode_sub(validator.item_validator, value_item) for value_item in validated_value] def encode_map(self, validator, value): validated_value = validator.validate(value) return { self.encode_sub(validator.key_validator, key): self.encode_sub(validator.value_validator, value) for key, value in validated_value.items() } def encode_nullable(self, validator, value): if value is None: return None return self.encode_sub(validator.validator, value) def encode_primitive(self, validator, value): if validator in self.alias_validators: self.alias_validators[validator](value) if isinstance(validator, bv.Void): return None elif isinstance(validator, bv.Timestamp): return _strftime(value, validator.format) elif isinstance(validator, bv.Bytes): if self.for_msgpack: return value else: return base64.b64encode(value).decode('ascii') elif isinstance(validator, bv.Integer) \ and isinstance(value, bool): # bool is sub-class of int so it passes Integer validation, # but we want the bool to be encoded as ``0`` or ``1``, rather # than ``False`` or ``True``, respectively return int(value) else: return value def encode_struct(self, validator, value): # Skip validation of fields with primitive data types because # they've already been validated on assignment d = collections.OrderedDict() # type: typing.Dict[str, typing.Any] all_fields = validator.definition._all_fields_ for extra_permission in self.caller_permissions.permissions: all_fields_name = '_all_{}_fields_'.format(extra_permission) all_fields = all_fields + getattr(validator.definition, all_fields_name, []) for field_name, field_validator in all_fields: try: field_value = getattr(value, field_name) except AttributeError as exc: raise bv.ValidationError(exc.args[0]) value_key = '_%s_value' % field_name if field_value is not None \ and getattr(value, value_key) is not bb.NOT_SET: # Only serialize struct fields that have been explicitly # set, even if there is a default try: d[field_name] = self.encode_sub(field_validator, field_value) except bv.ValidationError as exc: exc.add_parent(field_name) raise return d def encode_struct_tree(self, validator, value): assert type(value) in validator.definition._pytype_to_tag_and_subtype_, \ '%r is not a serializable subtype of %r.' % (type(value), validator.definition) tags, subtype = validator.definition._pytype_to_tag_and_subtype_[type(value)] assert len(tags) == 1, tags assert not isinstance(subtype, bv.StructTree), \ 'Cannot serialize type %r because it enumerates subtypes.' % subtype.definition if self.old_style: d = { tags[0]: self.encode_struct(subtype, value), } else: d = collections.OrderedDict() d['.tag'] = tags[0] d.update(self.encode_struct(subtype, value)) return d def encode_union(self, validator, value): if value._tag is None: raise bv.ValidationError('no tag set') if not validator.definition._is_tag_present(value._tag, self.caller_permissions): raise bv.ValidationError( "caller does not have access to '{}' tag".format(value._tag)) field_validator = validator.definition._get_val_data_type(value._tag, self.caller_permissions) is_none = isinstance(field_validator, bv.Void) \ or (isinstance(field_validator, bv.Nullable) and value._value is None) def encode_sub(sub_validator, sub_value, parent_tag): try: encoded_val = self.encode_sub(sub_validator, sub_value) except bv.ValidationError as exc: exc.add_parent(parent_tag) raise else: return encoded_val if self.old_style: if field_validator is None: return value._tag elif is_none: return value._tag else: encoded_val = encode_sub(field_validator, value._value, value._tag) return {value._tag: encoded_val} elif is_none: return {'.tag': value._tag} else: encoded_val = encode_sub(field_validator, value._value, value._tag) if isinstance(field_validator, bv.Nullable): # We've already checked for the null case above, # so now we're only interested in what the # wrapped validator is field_validator = field_validator.validator if isinstance(field_validator, bv.Struct) \ and not isinstance(field_validator, bv.StructTree): d = collections.OrderedDict() # type: typing.Dict[str, typing.Any] d['.tag'] = value._tag d.update(encoded_val) return d else: return collections.OrderedDict(( ('.tag', value._tag), (value._tag, encoded_val), )) # ------------------------------------------------------------------------ class StoneToJsonSerializer(StoneToPythonPrimitiveSerializer): def encode(self, validator, value): return json.dumps(super(StoneToJsonSerializer, self).encode(validator, value)) # -------------------------------------------------------------- # JSON Encoder # # These interfaces are preserved for backward compatibility and symmetry with deserialization # functions. def json_encode(data_type, obj, caller_permissions=None, alias_validators=None, old_style=False, should_redact=False): """Encodes an object into JSON based on its type. Args: data_type (Validator): Validator for obj. obj (object): Object to be serialized. caller_permissions (list): The list of raw-string caller permissions with which to serialize. alias_validators (Optional[Mapping[bv.Validator, Callable[[], None]]]): Custom validation functions. These must raise bv.ValidationError on failure. Returns: str: JSON-encoded object. This function will also do additional validation that wasn't done by the objects themselves: 1. The passed in obj may not have been validated with data_type yet. 2. If an object that should be a Struct was assigned to a field, its type has been validated, but the presence of all required fields hasn't been. 3. If an object that should be a Union was assigned to a field, whether or not a tag has been set has not been validated. 4. A list may have passed validation initially, but been mutated since. Example of serializing a struct to JSON: struct FileRef path String rev String > fr = FileRef() > fr.path = 'a/b/c' > fr.rev = '1234' > JsonEncoder.encode(fr) "{'path': 'a/b/c', 'rev': '1234'}" Example of serializing a union to JSON: union UploadMode add overwrite update FileRef > um = UploadMode() > um.set_add() > JsonEncoder.encode(um) '"add"' > um.update = fr > JsonEncoder.encode(um) "{'update': {'path': 'a/b/c', 'rev': '1234'}}" """ for_msgpack = False serializer = StoneToJsonSerializer( caller_permissions, alias_validators, for_msgpack, old_style, should_redact) return serializer.encode(data_type, obj) def json_compat_obj_encode(data_type, obj, caller_permissions=None, alias_validators=None, old_style=False, for_msgpack=False, should_redact=False): """Encodes an object into a JSON-compatible dict based on its type. Args: data_type (Validator): Validator for obj. obj (object): Object to be serialized. caller_permissions (list): The list of raw-string caller permissions with which to serialize. Returns: An object that when passed to json.dumps() will produce a string giving the JSON-encoded object. See json_encode() for additional information about validation. """ serializer = StoneToPythonPrimitiveSerializer( caller_permissions, alias_validators, for_msgpack, old_style, should_redact) return serializer.encode(data_type, obj) # -------------------------------------------------------------- # JSON Decoder class PythonPrimitiveToStoneDecoder(object): def __init__(self, caller_permissions, alias_validators, for_msgpack, old_style, strict): self.caller_permissions = (caller_permissions if caller_permissions else CallerPermissionsDefault()) self.alias_validators = alias_validators self.strict = strict self._old_style = old_style self._for_msgpack = for_msgpack @property def for_msgpack(self): """ EXPERIMENTAL: A flag associated with the serializer indicating whether objects produced by the ``encode`` method should be encoded for msgpack. """ return self._for_msgpack @property def old_style(self): """ A flag associated with the serializer indicating whether objects produced by the ``encode`` method should be encoded according to Dropbox's old or new API styles. """ return self._old_style def json_compat_obj_decode_helper(self, data_type, obj): """ See json_compat_obj_decode() for argument descriptions. """ if isinstance(data_type, bv.StructTree): return self.decode_struct_tree(data_type, obj) elif isinstance(data_type, bv.Struct): return self.decode_struct(data_type, obj) elif isinstance(data_type, bv.Union): if self.old_style: return self.decode_union_old(data_type, obj) else: return self.decode_union(data_type, obj) elif isinstance(data_type, bv.List): return self.decode_list( data_type, obj) elif isinstance(data_type, bv.Map): return self.decode_map( data_type, obj) elif isinstance(data_type, bv.Nullable): return self.decode_nullable( data_type, obj) elif isinstance(data_type, bv.Primitive): # Set validate to false because validation will be done by the # containing struct or union when the field is assigned. return self.make_stone_friendly(data_type, obj, False) else: raise AssertionError('Cannot handle type %r.' % data_type) def decode_struct(self, data_type, obj): """ The data_type argument must be a Struct. See json_compat_obj_decode() for argument descriptions. """ if obj is None and data_type.has_default(): return data_type.get_default() elif not isinstance(obj, dict): raise bv.ValidationError('expected object, got %s' % bv.generic_type_name(obj)) all_fields = data_type.definition._all_fields_ for extra_permission in self.caller_permissions.permissions: all_extra_fields = '_all_{}_fields_'.format(extra_permission) all_fields = all_fields + getattr(data_type.definition, all_extra_fields, []) if self.strict: all_field_names = data_type.definition._all_field_names_ for extra_permission in self.caller_permissions.permissions: all_extra_field_names = '_all_{}_field_names_'.format(extra_permission) all_field_names = all_field_names.union( getattr(data_type.definition, all_extra_field_names, {})) for key in obj: if (key not in all_field_names and not key.startswith('.tag')): raise bv.ValidationError("unknown field '%s'" % key) ins = data_type.definition() self.decode_struct_fields(ins, all_fields, obj) # Check that all required fields have been set. data_type.validate_fields_only_with_permissions(ins, self.caller_permissions) return ins def decode_struct_fields(self, ins, fields, obj): """ Args: ins: An instance of the class representing the data type being decoded. The object will have its fields set. fields: A tuple of (field_name: str, field_validator: Validator) obj (dict): JSON-compatible dict that is being decoded. strict (bool): See :func:`json_compat_obj_decode`. Returns: None: `ins` has its fields set based on the contents of `obj`. """ for name, field_data_type in fields: if name in obj: try: v = self.json_compat_obj_decode_helper(field_data_type, obj[name]) setattr(ins, name, v) except bv.ValidationError as e: e.add_parent(name) raise elif field_data_type.has_default(): setattr(ins, name, field_data_type.get_default()) def decode_union(self, data_type, obj): """ The data_type argument must be a Union. See json_compat_obj_decode() for argument descriptions. """ val = None if isinstance(obj, six.string_types): # Handles the shorthand format where the union is serialized as only # the string of the tag. tag = obj if data_type.definition._is_tag_present(tag, self.caller_permissions): val_data_type = data_type.definition._get_val_data_type( tag, self.caller_permissions) if not isinstance(val_data_type, (bv.Void, bv.Nullable)): raise bv.ValidationError( "expected object for '%s', got symbol" % tag) if tag == data_type.definition._catch_all: raise bv.ValidationError( "unexpected use of the catch-all tag '%s'" % tag) elif not self.strict and data_type.definition._catch_all: tag = data_type.definition._catch_all else: raise bv.ValidationError("unknown tag '%s'" % tag) elif isinstance(obj, dict): tag, val = self.decode_union_dict(data_type, obj) else: raise bv.ValidationError("expected string or object, got %s" % bv.generic_type_name(obj)) return data_type.definition(six.ensure_str(tag), val) def decode_union_dict(self, data_type, obj): if '.tag' not in obj: raise bv.ValidationError("missing '.tag' key") tag = obj['.tag'] if not isinstance(tag, six.string_types): raise bv.ValidationError( 'tag must be string, got %s' % bv.generic_type_name(tag)) if not data_type.definition._is_tag_present(tag, self.caller_permissions): if not self.strict and data_type.definition._catch_all: return data_type.definition._catch_all, None else: raise bv.ValidationError("unknown tag '%s'" % tag) if tag == data_type.definition._catch_all: raise bv.ValidationError( "unexpected use of the catch-all tag '%s'" % tag) val_data_type = data_type.definition._get_val_data_type(tag, self.caller_permissions) if isinstance(val_data_type, bv.Nullable): val_data_type = val_data_type.validator nullable = True else: nullable = False if isinstance(val_data_type, bv.Void): if self.strict: # In strict mode, ensure there are no extraneous keys set. In # non-strict mode, we accept that other keys may be set due to a # change of the void type to another. if tag in obj: if obj[tag] is not None: raise bv.ValidationError('expected null, got %s' % bv.generic_type_name(obj[tag])) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) val = None elif isinstance(val_data_type, (bv.Primitive, bv.List, bv.StructTree, bv.Union, bv.Map)): if tag in obj: raw_val = obj[tag] try: val = self.json_compat_obj_decode_helper(val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: # Check no other keys if nullable: val = None else: raise bv.ValidationError("missing '%s' key" % tag) for key in obj: if key != tag and key != '.tag': raise bv.ValidationError("unexpected key '%s'" % key) elif isinstance(val_data_type, bv.Struct): if nullable and len(obj) == 1: # only has a .tag key val = None else: # assume it's not null raw_val = obj try: val = self.json_compat_obj_decode_helper(val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: assert False, type(val_data_type) return tag, val def decode_union_old(self, data_type, obj): """ The data_type argument must be a Union. See json_compat_obj_decode() for argument descriptions. """ val = None if isinstance(obj, six.string_types): # Union member has no associated value tag = obj if data_type.definition._is_tag_present(tag, self.caller_permissions): val_data_type = data_type.definition._get_val_data_type(tag, self.caller_permissions) if not isinstance(val_data_type, (bv.Void, bv.Nullable)): raise bv.ValidationError( "expected object for '%s', got symbol" % tag) else: if not self.strict and data_type.definition._catch_all: tag = data_type.definition._catch_all else: raise bv.ValidationError("unknown tag '%s'" % tag) elif isinstance(obj, dict): # Union member has value if len(obj) != 1: raise bv.ValidationError('expected 1 key, got %s' % len(obj)) tag = list(obj)[0] raw_val = obj[tag] if data_type.definition._is_tag_present(tag, self.caller_permissions): val_data_type = data_type.definition._get_val_data_type(tag, self.caller_permissions) if isinstance(val_data_type, bv.Nullable) and raw_val is None: val = None elif isinstance(val_data_type, bv.Void): if raw_val is None or not self.strict: # If raw_val is None, then this is the more verbose # representation of a void union member. If raw_val isn't # None, then maybe the spec has changed, so check if we're # in strict mode. val = None else: raise bv.ValidationError('expected null, got %s' % bv.generic_type_name(raw_val)) else: try: val = self.json_compat_obj_decode_helper(val_data_type, raw_val) except bv.ValidationError as e: e.add_parent(tag) raise else: if not self.strict and data_type.definition._catch_all: tag = data_type.definition._catch_all else: raise bv.ValidationError("unknown tag '%s'" % tag) else: raise bv.ValidationError("expected string or object, got %s" % bv.generic_type_name(obj)) return data_type.definition(six.ensure_str(tag), val) def decode_struct_tree(self, data_type, obj): """ The data_type argument must be a StructTree. See json_compat_obj_decode() for argument descriptions. """ subtype = self.determine_struct_tree_subtype(data_type, obj) return self.decode_struct(subtype, obj) def determine_struct_tree_subtype(self, data_type, obj): """ Searches through the JSON-object-compatible dict using the data type definition to determine which of the enumerated subtypes `obj` is. """ if '.tag' not in obj: raise bv.ValidationError("missing '.tag' key") if not isinstance(obj['.tag'], six.string_types): raise bv.ValidationError('expected string, got %s' % bv.generic_type_name(obj['.tag']), parent='.tag') # Find the subtype the tags refer to full_tags_tuple = (obj['.tag'],) if full_tags_tuple in data_type.definition._tag_to_subtype_: subtype = data_type.definition._tag_to_subtype_[full_tags_tuple] if isinstance(subtype, bv.StructTree): raise bv.ValidationError("tag '%s' refers to non-leaf subtype" % ('.'.join(full_tags_tuple))) return subtype else: if self.strict: # In strict mode, the entirety of the tag hierarchy should # point to a known subtype. raise bv.ValidationError("unknown subtype '%s'" % '.'.join(full_tags_tuple)) else: # If subtype was not found, use the base. if data_type.definition._is_catch_all_: return data_type else: raise bv.ValidationError( "unknown subtype '%s' and '%s' is not a catch-all" % ('.'.join(full_tags_tuple), data_type.definition.__name__)) def decode_list(self, data_type, obj): """ The data_type argument must be a List. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, list): raise bv.ValidationError( 'expected list, got %s' % bv.generic_type_name(obj)) return [ self.json_compat_obj_decode_helper(data_type.item_validator, item) for item in obj] def decode_map(self, data_type, obj): """ The data_type argument must be a Map. See json_compat_obj_decode() for argument descriptions. """ if not isinstance(obj, dict): raise bv.ValidationError( 'expected dict, got %s' % bv.generic_type_name(obj)) return { self.json_compat_obj_decode_helper(data_type.key_validator, key): self.json_compat_obj_decode_helper(data_type.value_validator, value) for key, value in obj.items() } def decode_nullable(self, data_type, obj): """ The data_type argument must be a Nullable. See json_compat_obj_decode() for argument descriptions. """ if obj is not None: return self.json_compat_obj_decode_helper(data_type.validator, obj) else: return None def make_stone_friendly(self, data_type, val, validate): """ Convert a Python object to a type that will pass validation by its validator. Validation by ``alias_validators`` is performed even if ``validate`` is false. """ if isinstance(data_type, bv.Timestamp): try: ret = datetime.datetime.strptime(val, data_type.format) except (TypeError, ValueError) as e: raise bv.ValidationError(e.args[0]) elif isinstance(data_type, bv.Bytes): if self.for_msgpack: if isinstance(val, six.text_type): ret = val.encode('utf-8') else: ret = val else: try: ret = base64.b64decode(val) except (TypeError, binascii.Error): raise bv.ValidationError('invalid base64-encoded bytes') elif isinstance(data_type, bv.Void): if self.strict and val is not None: raise bv.ValidationError("expected null, got value") return None else: if validate: if self.caller_permissions.permissions: data_type.validate_with_permissions(val, self.caller_permissions) else: data_type.validate(val) ret = val if self.alias_validators is not None and data_type in self.alias_validators: self.alias_validators[data_type](ret) return ret def json_decode(data_type, serialized_obj, caller_permissions=None, alias_validators=None, strict=True, old_style=False): """Performs the reverse operation of json_encode. Args: data_type (Validator): Validator for serialized_obj. serialized_obj (str): The JSON string to deserialize. caller_permissions (list): The list of raw-string caller permissions with which to serialize. alias_validators (Optional[Mapping[bv.Validator, Callable[[], None]]]): Custom validation functions. These must raise bv.ValidationError on failure. strict (bool): If strict, then unknown struct fields will raise an error, and unknown union variants will raise an error even if a catch all field is specified. strict should only be used by a recipient of serialized JSON if it's guaranteed that its Stone specs are at least as recent as the senders it receives messages from. Returns: The returned object depends on the input data_type. - Boolean -> bool - Bytes -> bytes - Float -> float - Integer -> long - List -> list - Map -> dict - Nullable -> None or its wrapped type. - String -> unicode (PY2) or str (PY3) - Struct -> An instance of its definition attribute. - Timestamp -> datetime.datetime - Union -> An instance of its definition attribute. """ try: deserialized_obj = json.loads(serialized_obj) except ValueError: raise bv.ValidationError('could not decode input as JSON') else: return json_compat_obj_decode( data_type, deserialized_obj, caller_permissions=caller_permissions, alias_validators=alias_validators, strict=strict, old_style=old_style) def json_compat_obj_decode(data_type, obj, caller_permissions=None, alias_validators=None, strict=True, old_style=False, for_msgpack=False): """ Decodes a JSON-compatible object based on its data type into a representative Python object. Args: data_type (Validator): Validator for serialized_obj. obj: The JSON-compatible object to decode based on data_type. caller_permissions (list): The list of raw-string caller permissions with which to serialize. strict (bool): If strict, then unknown struct fields will raise an error, and unknown union variants will raise an error even if a catch all field is specified. See json_decode() for more. Returns: See json_decode(). """ decoder = PythonPrimitiveToStoneDecoder(caller_permissions, alias_validators, for_msgpack, old_style, strict) if isinstance(data_type, bv.Primitive): return decoder.make_stone_friendly( data_type, obj, True) else: return decoder.json_compat_obj_decode_helper( data_type, obj) # Adapted from: # http://code.activestate.com/recipes/306860-proleptic-gregorian-dates-and-strftime-before-1900/ # Remove the unsupposed "%s" command. But don't do it if there's an odd # number of %s before the s because those are all escaped. Can't simply # remove the s because the result of %sY should be %Y if %s isn't # supported, not the 4 digit year. _ILLEGAL_S = re.compile(r'((^|[^%])(%%)*%s)') def _findall(text, substr): # Also finds overlaps sites = [] i = 0 while 1: j = text.find(substr, i) if j == -1: break sites.append(j) i = j + 1 return sites # Every 28 years the calendar repeats, except through century leap years # where it's 6 years. But only if you're using the Gregorian calendar. ;) def _strftime(dt, fmt): try: return dt.strftime(fmt) except ValueError: if not six.PY2 or dt.year > 1900: raise if _ILLEGAL_S.search(fmt): raise TypeError("This strftime implementation does not handle %s") year = dt.year # For every non-leap year century, advance by 6 years to get into the # 28-year repeat cycle delta = 2000 - year off = 6 * (delta // 100 + delta // 400) year = year + off # Move to around the year 2000 year = year + ((2000 - year) // 28) * 28 timetuple = dt.timetuple() s1 = time.strftime(fmt, (year,) + timetuple[1:]) sites1 = _findall(s1, str(year)) s2 = time.strftime(fmt, (year + 28,) + timetuple[1:]) sites2 = _findall(s2, str(year + 28)) sites = [] for site in sites1: if site in sites2: sites.append(site) s = s1 syear = '%4d' % (dt.year,) for site in sites: s = s[:site] + syear + s[site + 4:] return s try: import msgpack except ImportError: pass else: msgpack_compat_obj_encode = functools.partial(json_compat_obj_encode, for_msgpack=True) def msgpack_encode(data_type, obj): return msgpack.dumps( msgpack_compat_obj_encode(data_type, obj), encoding='utf-8') msgpack_compat_obj_decode = functools.partial(json_compat_obj_decode, for_msgpack=True) def msgpack_decode( data_type, serialized_obj, alias_validators=None, strict=True): # We decode everything as utf-8 because we want all object keys to be # unicode. Otherwise, we need to do a lot more refactoring to make # json/msgpack share the same code. We expect byte arrays to fail # decoding, but when they don't, we have to convert them to bytes. deserialized_obj = msgpack.loads( serialized_obj, encoding='utf-8', unicode_errors='ignore') return msgpack_compat_obj_decode( data_type, deserialized_obj, alias_validators, strict) stone-3.3.1/stone/backends/python_rsrc/stone_validators.py000066400000000000000000000624731417406541500240770ustar00rootroot00000000000000""" Defines classes to represent each Stone type in Python. These classes should be used to validate Python objects and normalize them for a given type. The data types defined here should not be specific to an RPC or serialization format. """ from __future__ import absolute_import, unicode_literals import datetime import hashlib import math import numbers import re from abc import ABCMeta, abstractmethod import six _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # See if six.PY3: _binary_types = (bytes, memoryview) # noqa: E501,F821 # pylint: disable=undefined-variable,useless-suppression else: _binary_types = (bytes, buffer) # noqa: E501,F821 # pylint: disable=undefined-variable,useless-suppression class ValidationError(Exception): """Raised when a value doesn't pass validation by its validator.""" def __init__(self, message, parent=None): """ Args: message (str): Error message detailing validation failure. parent (str): Adds the parent as the closest reference point for the error. Use :meth:`add_parent` to add more. """ super(ValidationError, self).__init__(message) self.message = message self._parents = [] if parent: self._parents.append(parent) def add_parent(self, parent): """ Args: parent (str): Adds the parent to the top of the tree of references that lead to the validator that failed. """ self._parents.append(parent) def __str__(self): """ Returns: str: A descriptive message of the validation error that may also include the path to the validator that failed. """ if self._parents: return '{}: {}'.format('.'.join(self._parents[::-1]), self.message) else: return self.message def __repr__(self): # Not a perfect repr, but includes the error location information. return 'ValidationError(%r)' % six.text_type(self) def type_name_with_module(t): # type: (typing.Type[typing.Any]) -> typing.Any return '%s.%s' % (t.__module__, t.__name__) def generic_type_name(v): # type: (typing.Any) -> typing.Any """Return a descriptive type name that isn't Python specific. For example, an int value will return 'integer' rather than 'int'.""" if isinstance(v, bool): # Must come before any numbers checks since booleans are integers too return 'boolean' elif isinstance(v, numbers.Integral): # Must come before real numbers check since integrals are reals too return 'integer' elif isinstance(v, numbers.Real): return 'float' elif isinstance(v, (tuple, list)): return 'list' elif isinstance(v, six.string_types): return 'string' elif v is None: return 'null' else: return type_name_with_module(type(v)) class Validator(six.with_metaclass(ABCMeta, object)): """All primitive and composite data types should be a subclass of this.""" __slots__ = ("_redact",) @abstractmethod def validate(self, val): """Validates that val is of this data type. Returns: A normalized value if validation succeeds. Raises: ValidationError """ def has_default(self): return False def get_default(self): raise AssertionError('No default available.') class Primitive(Validator): # pylint: disable=abstract-method """A basic type that is defined by Stone.""" __slots__ = () class Boolean(Primitive): __slots__ = () def validate(self, val): if not isinstance(val, bool): raise ValidationError('%r is not a valid boolean' % val) return val class Integer(Primitive): """ Do not use this class directly. Extend it and specify a 'default_minimum' and 'default_maximum' value as class variables for a more restrictive integer range. """ __slots__ = ("minimum", "maximum") default_minimum = None # type: typing.Optional[int] default_maximum = None # type: typing.Optional[int] def __init__(self, min_value=None, max_value=None): """ A more restrictive minimum or maximum value can be specified than the range inherent to the defined type. """ if min_value is not None: assert isinstance(min_value, numbers.Integral), \ 'min_value must be an integral number' assert min_value >= self.default_minimum, \ 'min_value cannot be less than the minimum value for this ' \ 'type (%d < %d)' % (min_value, self.default_minimum) self.minimum = min_value else: self.minimum = self.default_minimum if max_value is not None: assert isinstance(max_value, numbers.Integral), \ 'max_value must be an integral number' assert max_value <= self.default_maximum, \ 'max_value cannot be greater than the maximum value for ' \ 'this type (%d < %d)' % (max_value, self.default_maximum) self.maximum = max_value else: self.maximum = self.default_maximum def validate(self, val): if not isinstance(val, numbers.Integral): raise ValidationError('expected integer, got %s' % generic_type_name(val)) elif not (self.minimum <= val <= self.maximum): raise ValidationError('%d is not within range [%d, %d]' % (val, self.minimum, self.maximum)) return val def __repr__(self): return '%s()' % self.__class__.__name__ class Int32(Integer): __slots__ = () default_minimum = -2**31 default_maximum = 2**31 - 1 class UInt32(Integer): __slots__ = () default_minimum = 0 default_maximum = 2**32 - 1 class Int64(Integer): __slots__ = () default_minimum = -2**63 default_maximum = 2**63 - 1 class UInt64(Integer): __slots__ = () default_minimum = 0 default_maximum = 2**64 - 1 class Real(Primitive): """ Do not use this class directly. Extend it and optionally set a 'default_minimum' and 'default_maximum' value to enforce a range that's a subset of the Python float implementation. Python floats are doubles. """ __slots__ = ("minimum", "maximum") default_minimum = None # type: typing.Optional[float] default_maximum = None # type: typing.Optional[float] def __init__(self, min_value=None, max_value=None): """ A more restrictive minimum or maximum value can be specified than the range inherent to the defined type. """ if min_value is not None: assert isinstance(min_value, numbers.Real), \ 'min_value must be a real number' if not isinstance(min_value, float): try: min_value = float(min_value) except OverflowError: raise AssertionError('min_value is too small for a float') if self.default_minimum is not None and min_value < self.default_minimum: raise AssertionError('min_value cannot be less than the ' 'minimum value for this type (%f < %f)' % (min_value, self.default_minimum)) self.minimum = min_value else: self.minimum = self.default_minimum if max_value is not None: assert isinstance(max_value, numbers.Real), \ 'max_value must be a real number' if not isinstance(max_value, float): try: max_value = float(max_value) except OverflowError: raise AssertionError('max_value is too large for a float') if self.default_maximum is not None and max_value > self.default_maximum: raise AssertionError('max_value cannot be greater than the ' 'maximum value for this type (%f < %f)' % (max_value, self.default_maximum)) self.maximum = max_value else: self.maximum = self.default_maximum def validate(self, val): if not isinstance(val, numbers.Real): raise ValidationError('expected real number, got %s' % generic_type_name(val)) if not isinstance(val, float): # This checks for the case where a number is passed in with a # magnitude larger than supported by float64. try: val = float(val) except OverflowError: raise ValidationError('too large for float') if math.isnan(val) or math.isinf(val): raise ValidationError('%f values are not supported' % val) if self.minimum is not None and val < self.minimum: raise ValidationError('%f is not greater than %f' % (val, self.minimum)) if self.maximum is not None and val > self.maximum: raise ValidationError('%f is not less than %f' % (val, self.maximum)) return val def __repr__(self): return '%s()' % self.__class__.__name__ class Float32(Real): __slots__ = () # Maximum and minimums from the IEEE 754-1985 standard default_minimum = -3.40282 * 10**38 default_maximum = 3.40282 * 10**38 class Float64(Real): __slots__ = () class String(Primitive): """Represents a unicode string.""" __slots__ = ("min_length", "max_length", "pattern", "pattern_re") def __init__(self, min_length=None, max_length=None, pattern=None): if min_length is not None: assert isinstance(min_length, numbers.Integral), \ 'min_length must be an integral number' assert min_length >= 0, 'min_length must be >= 0' if max_length is not None: assert isinstance(max_length, numbers.Integral), \ 'max_length must be an integral number' assert max_length > 0, 'max_length must be > 0' if min_length and max_length: assert max_length >= min_length, 'max_length must be >= min_length' if pattern is not None: assert isinstance(pattern, six.string_types), \ 'pattern must be a string' self.min_length = min_length self.max_length = max_length self.pattern = pattern self.pattern_re = None if pattern: try: self.pattern_re = re.compile(r"\A(?:" + pattern + r")\Z") except re.error as e: raise AssertionError('Regex {!r} failed: {}'.format( pattern, e.args[0])) def validate(self, val): """ A unicode string of the correct length and pattern will pass validation. In PY2, we enforce that a str type must be valid utf-8, and a unicode string will be returned. """ if not isinstance(val, six.string_types): raise ValidationError("'%s' expected to be a string, got %s" % (val, generic_type_name(val))) if not six.PY3 and isinstance(val, str): try: val = val.decode('utf-8') except UnicodeDecodeError: raise ValidationError("'%s' was not valid utf-8") if self.max_length is not None and len(val) > self.max_length: raise ValidationError("'%s' must be at most %d characters, got %d" % (val, self.max_length, len(val))) if self.min_length is not None and len(val) < self.min_length: raise ValidationError("'%s' must be at least %d characters, got %d" % (val, self.min_length, len(val))) if self.pattern and not self.pattern_re.match(val): raise ValidationError("'%s' did not match pattern '%s'" % (val, self.pattern)) return val class Bytes(Primitive): __slots__ = ("min_length", "max_length") def __init__(self, min_length=None, max_length=None): if min_length is not None: assert isinstance(min_length, numbers.Integral), \ 'min_length must be an integral number' assert min_length >= 0, 'min_length must be >= 0' if max_length is not None: assert isinstance(max_length, numbers.Integral), \ 'max_length must be an integral number' assert max_length > 0, 'max_length must be > 0' if min_length is not None and max_length is not None: assert max_length >= min_length, 'max_length must be >= min_length' self.min_length = min_length self.max_length = max_length def validate(self, val): if not isinstance(val, _binary_types): raise ValidationError("expected bytes type, got %s" % generic_type_name(val)) elif self.max_length is not None and len(val) > self.max_length: raise ValidationError("'%s' must have at most %d bytes, got %d" % (val, self.max_length, len(val))) elif self.min_length is not None and len(val) < self.min_length: raise ValidationError("'%s' has fewer than %d bytes, got %d" % (val, self.min_length, len(val))) return val class Timestamp(Primitive): """Note that while a format is specified, it isn't used in validation since a native Python datetime object is preferred. The format, however, can and should be used by serializers.""" __slots__ = ("format",) def __init__(self, fmt): """fmt must be composed of format codes that the C standard (1989) supports, most notably in its strftime() function.""" assert isinstance(fmt, six.text_type), 'format must be a string' self.format = fmt def validate(self, val): if not isinstance(val, datetime.datetime): raise ValidationError('expected timestamp, got %s' % generic_type_name(val)) elif val.tzinfo is not None and \ val.tzinfo.utcoffset(val).total_seconds() != 0: raise ValidationError('timestamp should have either a UTC ' 'timezone or none set at all') return val class Composite(Validator): # pylint: disable=abstract-method """Validator for a type that builds on other primitive and composite types.""" __slots__ = () class List(Composite): """Assumes list contents are homogeneous with respect to types.""" __slots__ = ("item_validator", "min_items", "max_items") def __init__(self, item_validator, min_items=None, max_items=None): """Every list item will be validated with item_validator.""" self.item_validator = item_validator if min_items is not None: assert isinstance(min_items, numbers.Integral), \ 'min_items must be an integral number' assert min_items >= 0, 'min_items must be >= 0' if max_items is not None: assert isinstance(max_items, numbers.Integral), \ 'max_items must be an integral number' assert max_items > 0, 'max_items must be > 0' if min_items is not None and max_items is not None: assert max_items >= min_items, 'max_items must be >= min_items' self.min_items = min_items self.max_items = max_items def validate(self, val): if not isinstance(val, (tuple, list)): raise ValidationError('%r is not a valid list' % val) elif self.max_items is not None and len(val) > self.max_items: raise ValidationError('%r has more than %s items' % (val, self.max_items)) elif self.min_items is not None and len(val) < self.min_items: raise ValidationError('%r has fewer than %s items' % (val, self.min_items)) return [self.item_validator.validate(item) for item in val] class Map(Composite): """Assumes map keys and values are homogeneous with respect to types.""" __slots__ = ("key_validator", "value_validator") def __init__(self, key_validator, value_validator): """ Every Map key/value pair will be validated with item_validator. key validators must be a subclass of a String validator """ self.key_validator = key_validator self.value_validator = value_validator def validate(self, val): if not isinstance(val, dict): raise ValidationError('%r is not a valid dict' % val) return { self.key_validator.validate(key): self.value_validator.validate(value) for key, value in val.items() } class Struct(Composite): __slots__ = ("definition",) def __init__(self, definition): """ Args: definition (class): A generated class representing a Stone struct from a spec. Must have a _fields_ attribute with the following structure: _fields_ = [(field_name, validator), ...] where field_name: Name of the field (str). validator: Validator object. """ super(Struct, self).__init__() self.definition = definition def validate(self, val): """ For a val to pass validation, val must be of the correct type and have all required fields present. """ self.validate_type_only(val) self.validate_fields_only(val) return val def validate_with_permissions(self, val, caller_permissions): """ For a val to pass validation, val must be of the correct type and have all required permissioned fields present. Should only be called for callers with extra permissions. """ self.validate(val) self.validate_fields_only_with_permissions(val, caller_permissions) return val def validate_fields_only(self, val): """ To pass field validation, no required field should be missing. This method assumes that the contents of each field have already been validated on assignment, so it's merely a presence check. FIXME(kelkabany): Since the definition object does not maintain a list of which fields are required, all fields are scanned. """ for field_name in self.definition._all_field_names_: if not hasattr(val, field_name): raise ValidationError("missing required field '%s'" % field_name) def validate_fields_only_with_permissions(self, val, caller_permissions): """ To pass field validation, no required field should be missing. This method assumes that the contents of each field have already been validated on assignment, so it's merely a presence check. Should only be called for callers with extra permissions. """ self.validate_fields_only(val) # check if type has been patched for extra_permission in caller_permissions.permissions: all_field_names = '_all_{}_field_names_'.format(extra_permission) for field_name in getattr(self.definition, all_field_names, set()): if not hasattr(val, field_name): raise ValidationError("missing required field '%s'" % field_name) def validate_type_only(self, val): """ Use this when you only want to validate that the type of an object is correct, but not yet validate each field. """ # Since the definition maintains the list of fields for serialization, # we're okay with a subclass that might have extra information. This # makes it easier to return one subclass for two routes, one of which # relies on the parent class. if not isinstance(val, self.definition): raise ValidationError('expected type %s, got %s' % ( type_name_with_module(self.definition), generic_type_name(val), ), ) def has_default(self): return not self.definition._has_required_fields def get_default(self): assert not self.definition._has_required_fields, 'No default available.' return self.definition() class StructTree(Struct): """Validator for structs with enumerated subtypes. NOTE: validate_fields_only() validates the fields known to this base struct, but does not do any validation specific to the subtype. """ __slots__ = () # See PyCQA/pylint#1043 for why this is disabled; this should show up # as a usless-suppression (and can be removed) once a fix is released def __init__(self, definition): # pylint: disable=useless-super-delegation super(StructTree, self).__init__(definition) class Union(Composite): __slots__ = ("definition",) def __init__(self, definition): """ Args: definition (class): A generated class representing a Stone union from a spec. Must have a _tagmap attribute with the following structure: _tagmap = {field_name: validator, ...} where field_name (str): Tag name. validator (Validator): Tag value validator. """ self.definition = definition def validate(self, val): """ For a val to pass validation, it must have a _tag set. This assumes that the object validated that _tag is a valid tag, and that any associated value has also been validated. """ self.validate_type_only(val) if not hasattr(val, '_tag') or val._tag is None: raise ValidationError('no tag set') return val def validate_type_only(self, val): """ Use this when you only want to validate that the type of an object is correct, but not yet validate each field. We check whether val is a Python parent class of the definition. This is because Union subtyping works in the opposite direction of Python inheritance. For example, if a union U2 extends U1 in Python, this validator will accept U1 in places where U2 is expected. """ if not issubclass(self.definition, type(val)): raise ValidationError('expected type %s or subtype, got %s' % ( type_name_with_module(self.definition), generic_type_name(val), ), ) class Void(Primitive): __slots__ = () def validate(self, val): if val is not None: raise ValidationError('expected NoneType, got %s' % generic_type_name(val)) def has_default(self): return True def get_default(self): return None class Nullable(Validator): __slots__ = ("validator",) def __init__(self, validator): super(Nullable, self).__init__() assert isinstance(validator, (Primitive, Composite)), \ 'validator must be for a primitive or composite type' assert not isinstance(validator, Nullable), \ 'nullables cannot be stacked' assert not isinstance(validator, Void), \ 'void cannot be made nullable' self.validator = validator def validate(self, val): if val is None: return else: return self.validator.validate(val) def validate_type_only(self, val): """Use this only if Nullable is wrapping a Composite.""" if val is None: return else: return self.validator.validate_type_only(val) def has_default(self): return True def get_default(self): return None class Redactor(object): __slots__ = ("regex",) def __init__(self, regex): """ Args: regex: What parts of the field to redact. """ self.regex = regex @abstractmethod def apply(self, val): """Redacts information from annotated field. Returns: A redacted version of the string provided. """ def _get_matches(self, val): if not self.regex: return None try: return re.search(self.regex, val) except TypeError: return None class HashRedactor(Redactor): __slots__ = () def apply(self, val): matches = self._get_matches(val) val_to_hash = str(val) if isinstance(val, int) or isinstance(val, float) else val try: # add string literal to ensure unicode hashed = hashlib.md5(val_to_hash.encode('utf-8')).hexdigest() + '' except [AttributeError, ValueError]: hashed = None if matches: blotted = '***'.join(matches.groups()) if hashed: return '{} ({})'.format(hashed, blotted) return blotted return hashed class BlotRedactor(Redactor): __slots__ = () def apply(self, val): matches = self._get_matches(val) if matches: return '***'.join(matches.groups()) return '********' stone-3.3.1/stone/backends/python_type_mapping.py000066400000000000000000000076071417406541500222400ustar00rootroot00000000000000from stone.ir import ( Alias, ApiNamespace, DataType, List, Map, Nullable, Timestamp, UserDefined, is_alias, is_boolean_type, is_bytes_type, is_float_type, is_integer_type, is_list_type, is_map_type, is_nullable_type, is_string_type, is_timestamp_type, is_user_defined_type, is_void_type, ) from stone.backends.python_helpers import class_name_for_data_type, fmt_namespace from stone.ir.data_types import String from stone.typing_hacks import cast MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression DataTypeCls = typing.Type[DataType] # Unfortunately these are codependent, so I'll weakly type the Dict in Callback Callback = typing.Callable[ [ApiNamespace, DataType, typing.Dict[typing.Any, typing.Any]], typing.Text ] OverrideDefaultTypesDict = typing.Dict[DataTypeCls, Callback] else: OverrideDefaultTypesDict = "OverrideDefaultTypesDict" def map_stone_type_to_python_type(ns, data_type, override_dict=None): # type: (ApiNamespace, DataType, typing.Optional[OverrideDefaultTypesDict]) -> typing.Text """ Args: override_dict: lets you override the default behavior for a given type by hooking into a callback. (Currently only hooked up for stone's List and Nullable) """ override_dict = override_dict or {} if is_string_type(data_type): string_override = override_dict.get(String, None) if string_override: return string_override(ns, data_type, override_dict) return 'str' elif is_bytes_type(data_type): return 'bytes' elif is_boolean_type(data_type): return 'bool' elif is_float_type(data_type): return 'float' elif is_integer_type(data_type): return 'int' elif is_void_type(data_type): return 'None' elif is_timestamp_type(data_type): timestamp_override = override_dict.get(Timestamp, None) if timestamp_override: return timestamp_override(ns, data_type, override_dict) return 'datetime.datetime' elif is_alias(data_type): alias_type = cast(Alias, data_type) return map_stone_type_to_python_type(ns, alias_type.data_type, override_dict) elif is_user_defined_type(data_type): user_defined_type = cast(UserDefined, data_type) class_name = class_name_for_data_type(user_defined_type) if user_defined_type.namespace.name != ns.name: return '{}.{}'.format( fmt_namespace(user_defined_type.namespace.name), class_name) else: return class_name elif is_list_type(data_type): list_type = cast(List, data_type) if List in override_dict: return override_dict[List](ns, list_type.data_type, override_dict) # PyCharm understands this description format for a list return 'list of [{}]'.format( map_stone_type_to_python_type(ns, list_type.data_type, override_dict) ) elif is_map_type(data_type): map_type = cast(Map, data_type) if Map in override_dict: return override_dict[Map]( ns, data_type, override_dict ) return 'dict of [{}:{}]'.format( map_stone_type_to_python_type(ns, map_type.key_data_type, override_dict), map_stone_type_to_python_type(ns, map_type.value_data_type, override_dict) ) elif is_nullable_type(data_type): nullable_type = cast(Nullable, data_type) if Nullable in override_dict: return override_dict[Nullable](ns, nullable_type.data_type, override_dict) return 'Optional[{}]'.format( map_stone_type_to_python_type(ns, nullable_type.data_type, override_dict) ) else: raise TypeError('Unknown data type %r' % data_type) stone-3.3.1/stone/backends/python_type_stubs.py000066400000000000000000000444001417406541500217350ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import argparse from six import StringIO from stone.backend import CodeBackend from stone.backends.python_helpers import ( check_route_name_conflict, class_name_for_annotation_type, class_name_for_data_type, emit_pass_if_nothing_emitted, fmt_func, fmt_namespace, fmt_var, generate_imports_for_referenced_namespaces, generate_module_header, validators_import_with_type_ignore, ) from stone.backends.python_type_mapping import ( map_stone_type_to_python_type, OverrideDefaultTypesDict, ) from stone.ir import ( Alias, AnnotationType, Api, ApiNamespace, DataType, is_nullable_type, is_struct_type, is_union_type, is_user_defined_type, is_void_type, List, Map, Nullable, Struct, Timestamp, Union, unwrap_aliases, ) from stone.ir.data_types import String from stone.typing_hacks import cast _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class ImportTracker(object): def __init__(self): # type: () -> None self.cur_namespace_typing_imports = set() # type: typing.Set[typing.Text] self.cur_namespace_adhoc_imports = set() # type: typing.Set[typing.Text] def clear(self): # type: () -> None self.cur_namespace_typing_imports.clear() self.cur_namespace_adhoc_imports.clear() def _register_typing_import(self, s): # type: (typing.Text) -> None """ Denotes that we need to import something specifically from the `typing` module. For example, _register_typing_import("Optional") """ self.cur_namespace_typing_imports.add(s) def _register_adhoc_import(self, s): # type: (typing.Text) -> None """ Denotes an ad-hoc import. For example, _register_adhoc_import("import datetime") or _register_adhoc_import("from xyz import abc") """ self.cur_namespace_adhoc_imports.add(s) _cmdline_parser = argparse.ArgumentParser(prog='python-types-backend') _cmdline_parser.add_argument( '-p', '--package', type=str, required=True, help='Package prefix for absolute imports in generated files.', ) class PythonTypeStubsBackend(CodeBackend): """Generates Python modules to represent the input Stone spec.""" cmdline_parser = _cmdline_parser # Instance var of the current namespace being generated cur_namespace = None preserve_aliases = True import_tracker = ImportTracker() def __init__(self, *args, **kwargs): # type: (...) -> None super(PythonTypeStubsBackend, self).__init__(*args, **kwargs) self._pep_484_type_mapping_callbacks = self._get_pep_484_type_mapping_callbacks() def generate(self, api): # type: (Api) -> None """ Generates a module for each namespace. Each namespace will have Python classes to represent data types and routes in the Stone spec. """ for namespace in api.namespaces.values(): with self.output_to_relative_path('{}.pyi'.format(fmt_namespace(namespace.name))): self._generate_base_namespace_module(namespace) def _generate_base_namespace_module(self, namespace): # type: (ApiNamespace) -> None """Creates a module for the namespace. All data types and routes are represented as Python classes.""" self.cur_namespace = namespace self.import_tracker.clear() generate_module_header(self) self.emit_placeholder('imports_needed_for_typing') self.emit_raw(validators_import_with_type_ignore) # Generate import statements for all referenced namespaces. self._generate_imports_for_referenced_namespaces(namespace) self._generate_typevars() for annotation_type in namespace.annotation_types: self._generate_annotation_type_class(namespace, annotation_type) for data_type in namespace.linearize_data_types(): if isinstance(data_type, Struct): self._generate_struct_class(namespace, data_type) elif isinstance(data_type, Union): self._generate_union_class(namespace, data_type) else: raise TypeError('Cannot handle type %r' % type(data_type)) for alias in namespace.linearize_aliases(): self._generate_alias_definition(namespace, alias) self._generate_routes(namespace) self._generate_imports_needed_for_typing() def _generate_imports_for_referenced_namespaces(self, namespace): # type: (ApiNamespace) -> None assert self.args is not None generate_imports_for_referenced_namespaces( backend=self, namespace=namespace, package=self.args.package, insert_type_ignore=True, ) def _generate_typevars(self): # type: () -> None """ Creates type variables that are used by the type signatures for _process_custom_annotations. """ self.emit("T = TypeVar('T', bound=bb.AnnotationType)") self.emit("U = TypeVar('U')") self.import_tracker._register_typing_import('TypeVar') self.emit() def _generate_annotation_type_class(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None """Defines a Python class that represents an annotation type in Stone.""" self.emit('class {}(object):'.format(class_name_for_annotation_type(annotation_type, ns))) with self.indent(): self._generate_annotation_type_class_init(ns, annotation_type) self._generate_annotation_type_class_properties(ns, annotation_type) self.emit() def _generate_annotation_type_class_init(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None args = ['self'] for param in annotation_type.params: param_name = fmt_var(param.name, True) param_type = self.map_stone_type_to_pep484_type(ns, param.data_type) if not is_nullable_type(param.data_type): self.import_tracker._register_typing_import('Optional') param_type = 'Optional[{}]'.format(param_type) args.append( "{param_name}: {param_type} = ...".format( param_name=param_name, param_type=param_type)) self.generate_multiline_list(args, before='def __init__', after=' -> None: ...') self.emit() def _generate_annotation_type_class_properties(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None for param in annotation_type.params: prop_name = fmt_var(param.name, True) param_type = self.map_stone_type_to_pep484_type(ns, param.data_type) self.emit('@property') self.emit('def {prop_name}(self) -> {param_type}: ...'.format( prop_name=prop_name, param_type=param_type, )) self.emit() def _generate_struct_class(self, ns, data_type): # type: (ApiNamespace, Struct) -> None """Defines a Python class that represents a struct in Stone.""" self.emit(self._class_declaration_for_type(ns, data_type)) with self.indent(): self._generate_struct_class_init(ns, data_type) self._generate_struct_class_properties(ns, data_type) self._generate_struct_or_union_class_custom_annotations() self._generate_validator_for(data_type) self.emit() def _generate_validator_for(self, data_type): # type: (DataType) -> None cls_name = class_name_for_data_type(data_type) self.emit("{}_validator: bv.Validator = ...".format( cls_name )) def _generate_union_class(self, ns, data_type): # type: (ApiNamespace, Union) -> None self.emit(self._class_declaration_for_type(ns, data_type)) with self.indent(), emit_pass_if_nothing_emitted(self): self._generate_union_class_vars(ns, data_type) self._generate_union_class_is_set(data_type) self._generate_union_class_variant_creators(ns, data_type) self._generate_union_class_get_helpers(ns, data_type) self._generate_struct_or_union_class_custom_annotations() self._generate_validator_for(data_type) self.emit() def _generate_union_class_vars(self, ns, data_type): # type: (ApiNamespace, Union) -> None lineno = self.lineno # Generate stubs for class variables so that IDEs like PyCharms have an # easier time detecting their existence. for field in data_type.fields: if is_void_type(field.data_type): field_name = fmt_var(field.name) field_type = class_name_for_data_type(data_type, ns) self.emit('{field_name}: {field_type} = ...'.format( field_name=field_name, field_type=field_type, )) if lineno != self.lineno: self.emit() def _generate_union_class_is_set(self, union): # type: (Union) -> None for field in union.fields: field_name = fmt_func(field.name) self.emit('def is_{}(self) -> bool: ...'.format(field_name)) self.emit() def _generate_union_class_variant_creators(self, ns, data_type): # type: (ApiNamespace, Union) -> None """ Generate the following section in the 'union Shape' example: @classmethod def circle(cls, val: float) -> Shape: ... """ union_type = class_name_for_data_type(data_type) for field in data_type.fields: if not is_void_type(field.data_type): field_name_reserved_check = fmt_func(field.name, check_reserved=True) val_type = self.map_stone_type_to_pep484_type(ns, field.data_type) self.emit('@classmethod') self.emit('def {field_name}(cls, val: {val_type}) -> {union_type}: ...'.format( field_name=field_name_reserved_check, val_type=val_type, union_type=union_type, )) self.emit() def _generate_union_class_get_helpers(self, ns, data_type): # type: (ApiNamespace, Union) -> None """ Generates the following section in the 'union Shape' example: def get_circle(self) -> float: ... """ for field in data_type.fields: field_name = fmt_func(field.name) if not is_void_type(field.data_type): # generate getter for field val_type = self.map_stone_type_to_pep484_type(ns, field.data_type) self.emit('def get_{field_name}(self) -> {val_type}: ...'.format( field_name=field_name, val_type=val_type, )) self.emit() def _generate_alias_definition(self, namespace, alias): # type: (ApiNamespace, Alias) -> None self._generate_validator_for(alias) unwrapped_dt, _ = unwrap_aliases(alias) if is_user_defined_type(unwrapped_dt): # If the alias is to a composite type, we want to alias the # generated class as well. self.emit('{} = {}'.format( alias.name, class_name_for_data_type(alias.data_type, namespace))) def _class_declaration_for_type(self, ns, data_type): # type: (ApiNamespace, typing.Union[Struct, Union]) -> typing.Text assert is_user_defined_type(data_type), \ 'Expected struct, got %r' % type(data_type) if data_type.parent_type: extends = class_name_for_data_type(data_type.parent_type, ns) else: if is_struct_type(data_type): # Use a handwritten base class extends = 'bb.Struct' elif is_union_type(data_type): extends = 'bb.Union' else: extends = 'object' return 'class {}({}):'.format( class_name_for_data_type(data_type), extends) def _generate_struct_class_init(self, ns, struct): # type: (ApiNamespace, Struct) -> None args = ["self"] for field in struct.all_fields: field_name_reserved_check = fmt_var(field.name, True) field_type = self.map_stone_type_to_pep484_type(ns, field.data_type) if field.has_default: self.import_tracker._register_typing_import('Optional') field_type = 'Optional[{}]'.format(field_type) args.append("{field_name}: {field_type} = ...".format( field_name=field_name_reserved_check, field_type=field_type)) self.generate_multiline_list(args, before='def __init__', after=' -> None: ...') def _generate_struct_class_properties(self, ns, struct): # type: (ApiNamespace, Struct) -> None to_emit = [] # type: typing.List[typing.Text] for field in struct.all_fields: field_name_reserved_check = fmt_func(field.name, check_reserved=True) field_type = self.map_stone_type_to_pep484_type(ns, field.data_type) to_emit.append( "{}: bb.Attribute[{}] = ...".format(field_name_reserved_check, field_type) ) for s in to_emit: self.emit(s) def _generate_struct_or_union_class_custom_annotations(self): """ The _process_custom_annotations function allows client code to access custom annotations defined in the spec. """ self.emit('def _process_custom_annotations(') with self.indent(): self.emit('self,') self.emit('annotation_type: Type[T],') self.emit('field_path: Text,') self.emit('processor: Callable[[T, U], U],') self.import_tracker._register_typing_import('Type') self.import_tracker._register_typing_import('Text') self.import_tracker._register_typing_import('Callable') self.emit(') -> None: ...') self.emit() def _get_pep_484_type_mapping_callbacks(self): # type: () -> OverrideDefaultTypesDict """ Once-per-instance, generate a mapping from "List" -> return pep4848-compatible List[SomeType] "Nullable" -> return pep484-compatible Optional[SomeType] This is per-instance because we have to also call `self._register_typing_import`, because we need to potentially import some things. """ def upon_encountering_list(ns, data_type, override_dict): # type: (ApiNamespace, DataType, OverrideDefaultTypesDict) -> typing.Text self.import_tracker._register_typing_import("List") return "List[{}]".format( map_stone_type_to_python_type(ns, data_type, override_dict) ) def upon_encountering_map(ns, map_data_type, override_dict): # type: (ApiNamespace, DataType, OverrideDefaultTypesDict) -> typing.Text map_type = cast(Map, map_data_type) self.import_tracker._register_typing_import("Dict") return "Dict[{}, {}]".format( map_stone_type_to_python_type(ns, map_type.key_data_type, override_dict), map_stone_type_to_python_type(ns, map_type.value_data_type, override_dict) ) def upon_encountering_nullable(ns, data_type, override_dict): # type: (ApiNamespace, DataType, OverrideDefaultTypesDict) -> typing.Text self.import_tracker._register_typing_import("Optional") return "Optional[{}]".format( map_stone_type_to_python_type(ns, data_type, override_dict) ) def upon_encountering_timestamp( ns, data_type, override_dict ): # pylint: disable=unused-argument # type: (ApiNamespace, DataType, OverrideDefaultTypesDict) -> typing.Text self.import_tracker._register_adhoc_import("import datetime") return map_stone_type_to_python_type(ns, data_type) def upon_encountering_string( ns, data_type, override_dict ): # pylint: disable=unused-argument # type: (...) -> typing.Text self.import_tracker._register_typing_import("Text") return "Text" callback_dict = { List: upon_encountering_list, Map: upon_encountering_map, Nullable: upon_encountering_nullable, Timestamp: upon_encountering_timestamp, String: upon_encountering_string, } # type: OverrideDefaultTypesDict return callback_dict def map_stone_type_to_pep484_type(self, ns, data_type): # type: (ApiNamespace, DataType) -> typing.Text assert self._pep_484_type_mapping_callbacks return map_stone_type_to_python_type(ns, data_type, override_dict=self._pep_484_type_mapping_callbacks) def _generate_routes( self, namespace, # type: ApiNamespace ): # type: (...) -> None check_route_name_conflict(namespace) for route in namespace.routes: self.emit( "{method_name}: bb.Route = ...".format( method_name=fmt_func(route.name, version=route.version))) if namespace.routes: self.emit() def _generate_imports_needed_for_typing(self): # type: () -> None output_buffer = StringIO() with self.capture_emitted_output(output_buffer): if self.import_tracker.cur_namespace_typing_imports: self.emit("") self.emit('from typing import (') with self.indent(): for to_import in sorted(self.import_tracker.cur_namespace_typing_imports): self.emit("{},".format(to_import)) self.emit(')') if self.import_tracker.cur_namespace_adhoc_imports: self.emit("") for to_import in self.import_tracker.cur_namespace_adhoc_imports: self.emit(to_import) self.add_named_placeholder('imports_needed_for_typing', output_buffer.getvalue()) stone-3.3.1/stone/backends/python_types.py000066400000000000000000001414501417406541500207030ustar00rootroot00000000000000""" Backend for generating Python types that match the spec. """ from __future__ import absolute_import, division, print_function, unicode_literals import argparse import itertools import re _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression from stone.ir import AnnotationType, ApiNamespace from stone.ir import ( is_alias, is_boolean_type, is_composite_type, is_bytes_type, is_list_type, is_map_type, is_nullable_type, is_numeric_type, is_string_type, is_struct_type, is_tag_ref, is_timestamp_type, is_union_type, is_user_defined_type, is_void_type, RedactedBlot, RedactedHash, Struct, Union, unwrap, unwrap_aliases, unwrap_nullable, ) from stone.ir import DataType from stone.backend import CodeBackend from stone.backends.python_helpers import ( class_name_for_annotation_type, class_name_for_data_type, check_route_name_conflict, emit_pass_if_nothing_emitted, fmt_class, fmt_func, fmt_namespace, fmt_namespaced_var, fmt_obj, fmt_var, generate_imports_for_referenced_namespaces, generate_module_header, validators_import, ) from stone.backends.python_type_mapping import map_stone_type_to_python_type # Matches format of Stone doc tags doc_sub_tag_re = re.compile(':(?P[A-z]*):`(?P.*?)`') _cmdline_parser = argparse.ArgumentParser(prog='python-types-backend') _cmdline_parser.add_argument( '-r', '--route-method', help=('A string used to construct the location of a Python method for a ' 'given route; use {ns} as a placeholder for namespace name and ' '{route} for the route name. This is used to translate Stone doc ' 'references to routes to references in Python docstrings.'), ) _cmdline_parser.add_argument( '-p', '--package', type=str, required=True, help='Package prefix for absolute imports in generated files.', ) class PythonTypesBackend(CodeBackend): """Generates Python modules to represent the input Stone spec.""" cmdline_parser = _cmdline_parser # Instance var of the current namespace being generated cur_namespace = None # type: typing.Optional[ApiNamespace] preserve_aliases = True def generate(self, api): """ Generates a module for each namespace. Each namespace will have Python classes to represent data types and routes in the Stone spec. """ with self.output_to_relative_path('__init__.py', mode='ab'): pass with self.output_to_relative_path('stone_base.py'): self.emit("from stone.backends.python_rsrc.stone_base import *") with self.output_to_relative_path('stone_serializers.py'): self.emit("from stone.backends.python_rsrc.stone_serializers import *") with self.output_to_relative_path('stone_validators.py'): self.emit("from stone.backends.python_rsrc.stone_validators import *") for namespace in api.namespaces.values(): reserved_namespace_name = fmt_namespace(namespace.name) with self.output_to_relative_path('{}.py'.format(reserved_namespace_name)): self._generate_base_namespace_module(api, namespace) if reserved_namespace_name != namespace.name: with self.output_to_relative_path('{}.py'.format(namespace.name)): self._generate_dummy_namespace_module(reserved_namespace_name) def _generate_base_namespace_module(self, api, namespace): """Creates a module for the namespace. All data types and routes are represented as Python classes.""" self.cur_namespace = namespace generate_module_header(self) if namespace.doc is not None: self.emit('"""') self.emit_raw(self.process_doc(namespace.doc, self._docf)) self.emit('"""') self.emit() self.emit("from __future__ import unicode_literals") self.emit_raw(validators_import) # Generate import statements for all referenced namespaces. self._generate_imports_for_referenced_namespaces(namespace) for annotation_type in namespace.annotation_types: self._generate_annotation_type_class(namespace, annotation_type) for data_type in namespace.linearize_data_types(): if isinstance(data_type, Struct): self._generate_struct_class(namespace, data_type) elif isinstance(data_type, Union): self._generate_union_class(namespace, data_type) else: raise TypeError('Cannot handle type %r' % type(data_type)) for alias in namespace.linearize_aliases(): self._generate_alias_definition(namespace, alias) # Generate the struct->subtype tag mapping at the end so that # references to later-defined subtypes don't cause errors. for data_type in namespace.linearize_data_types(): if is_struct_type(data_type): self._generate_struct_class_reflection_attributes( namespace, data_type) if data_type.has_enumerated_subtypes(): self._generate_enumerated_subtypes_tag_mapping( namespace, data_type) elif is_union_type(data_type): self._generate_union_class_reflection_attributes( namespace, data_type) self._generate_union_class_symbol_creators(data_type) for data_type in namespace.linearize_data_types(): if is_struct_type(data_type): self._generate_struct_attributes_defaults( namespace, data_type) self._generate_routes(api.route_schema, namespace) def _generate_dummy_namespace_module(self, reserved_namespace_name): generate_module_header(self) self.emit('# If you have issues importing this module because Python recognizes it as a ' 'keyword, use {} instead.'.format(reserved_namespace_name)) self.emit('from .{} import *'.format(reserved_namespace_name)) def _generate_alias_definition(self, namespace, alias): v = generate_validator_constructor(namespace, alias.data_type) if alias.doc: self.emit_wrapped_text( self.process_doc(alias.doc, self._docf), prefix='# ') validator_name = '{}_validator'.format(alias.name) self.emit('{} = {}'.format(validator_name, v)) if alias.redactor: self._generate_redactor(validator_name, alias.redactor) unwrapped_dt, _ = unwrap_aliases(alias) if is_user_defined_type(unwrapped_dt): # If the alias is to a composite type, we want to alias the # generated class as well. self.emit('{} = {}'.format( alias.name, class_name_for_data_type(alias.data_type, namespace))) def _generate_imports_for_referenced_namespaces(self, namespace): # type: (ApiNamespace) -> None assert self.args is not None generate_imports_for_referenced_namespaces( backend=self, namespace=namespace, package=self.args.package, ) def _docf(self, tag, val): """ Callback used as the handler argument to process_docs(). This converts Stone doc references to Sphinx-friendly annotations. """ if tag == 'type': return ':class:`{}`'.format(val) elif tag == 'route': if self.args.route_method: return ':meth:`%s`' % self.args.route_method.format( ns=self.cur_namespace.name, route=fmt_func(val)) else: return val elif tag == 'link': anchor, link = val.rsplit(' ', 1) return '`{} <{}>`_'.format(anchor, link) elif tag == 'val': if val == 'null': return 'None' elif val == 'true' or val == 'false': return '``{}``'.format(val.capitalize()) else: return val elif tag == 'field': return '``{}``'.format(val) else: raise RuntimeError('Unknown doc ref tag %r' % tag) def _python_type_mapping(self, ns, data_type): # type: (ApiNamespace, DataType) -> typing.Text """Map Stone data types to their most natural equivalent in Python for documentation purposes.""" return map_stone_type_to_python_type(ns, data_type) def _class_declaration_for_type(self, ns, data_type): assert is_user_defined_type(data_type), \ 'Expected struct, got %r' % type(data_type) if data_type.parent_type: extends = class_name_for_data_type(data_type.parent_type, ns) else: if is_struct_type(data_type): # Use a handwritten base class extends = 'bb.Struct' elif is_union_type(data_type): extends = 'bb.Union' else: extends = 'object' return 'class {}({}):'.format( class_name_for_data_type(data_type), extends) # # Annotation types # def _generate_annotation_type_class(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None """Defines a Python class that represents an annotation type in Stone.""" self.emit('class {}(bb.AnnotationType):'.format( class_name_for_annotation_type(annotation_type, ns))) with self.indent(): if annotation_type.has_documented_type_or_params(): self.emit('"""') if annotation_type.doc: self.emit_wrapped_text( self.process_doc(annotation_type.doc, self._docf)) if annotation_type.has_documented_params(): self.emit() for param in annotation_type.params: if not param.doc: continue self.emit_wrapped_text(':ivar {}: {}'.format( fmt_var(param.name, True), self.process_doc(param.doc, self._docf)), subsequent_prefix=' ') self.emit('"""') self.emit() self._generate_annotation_type_class_slots(annotation_type) self._generate_annotation_type_class_init(ns, annotation_type) self._generate_annotation_type_class_properties(ns, annotation_type) self.emit() def _generate_annotation_type_class_slots(self, annotation_type): # type: (AnnotationType) -> None with self.block('__slots__ =', delim=('[', ']')): for param in annotation_type.params: param_name = fmt_var(param.name, True) self.emit("'_{}',".format(param_name)) self.emit() def _generate_annotation_type_class_init(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None args = ['self'] for param in annotation_type.params: param_name = fmt_var(param.name, True) default_value = (self._generate_python_value(ns, param.default) if param.has_default else 'None') args.append('{}={}'.format(param_name, default_value)) self.generate_multiline_list(args, before='def __init__', after=':') with self.indent(): for param in annotation_type.params: self.emit('self._{0} = {0}'.format(fmt_var(param.name, True))) self.emit() def _generate_annotation_type_class_properties(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> None for param in annotation_type.params: param_name = fmt_var(param.name, True) prop_name = fmt_func(param.name, True) self.emit('@property') self.emit('def {}(self):'.format(prop_name)) with self.indent(): self.emit('"""') if param.doc: self.emit_wrapped_text( self.process_doc(param.doc, self._docf)) # Sphinx wants an extra line between the text and the # rtype declaration. self.emit() self.emit(':rtype: {}'.format( self._python_type_mapping(ns, param.data_type))) self.emit('"""') self.emit('return self._{}'.format(param_name)) self.emit() # # Struct Types # def _generate_struct_class(self, ns, data_type): # type: (ApiNamespace, Struct) -> None """Defines a Python class that represents a struct in Stone.""" self.emit(self._class_declaration_for_type(ns, data_type)) with self.indent(): if data_type.has_documented_type_or_fields(): self.emit('"""') if data_type.doc: self.emit_wrapped_text( self.process_doc(data_type.doc, self._docf)) if data_type.has_documented_fields(): self.emit() for field in data_type.fields: if not field.doc: continue self.emit_wrapped_text(':ivar {}: {}'.format( fmt_namespaced_var(ns.name, data_type.name, field.name), self.process_doc(field.doc, self._docf)), subsequent_prefix=' ') self.emit('"""') self.emit() self._generate_struct_class_slots(data_type) self._generate_struct_class_has_required_fields(data_type) self._generate_struct_class_init(data_type) self._generate_struct_class_properties(ns, data_type) self._generate_struct_class_custom_annotations(ns, data_type) if data_type.has_enumerated_subtypes(): validator = 'StructTree' else: validator = 'Struct' self.emit('{0}_validator = bv.{1}({0})'.format( class_name_for_data_type(data_type), validator, )) self.emit() def _func_args_from_dict(self, d): """Given a Python dictionary, creates a string representing arguments for invoking a function. All arguments with a value of None are ignored.""" filtered_d = self.filter_out_none_valued_keys(d) return ', '.join(['%s=%s' % (k, v) for k, v in filtered_d.items()]) def _generate_struct_class_slots(self, data_type): """Creates a slots declaration for struct classes. Slots are an optimization in Python. They reduce the memory footprint of instances since attributes cannot be added after declaration. """ with self.block('__slots__ =', delim=('[', ']')): for field in data_type.fields: field_name = fmt_var(field.name) self.emit("'_%s_value'," % field_name) self.emit() def _generate_struct_class_has_required_fields(self, data_type): has_required_fields = len(data_type.all_required_fields) > 0 self.emit('_has_required_fields = %r' % has_required_fields) self.emit() def _generate_struct_class_reflection_attributes(self, ns, data_type): """ Generates two class attributes: * _all_field_names_: Set of all field names including inherited fields. * _all_fields_: List of tuples, where each tuple is (name, validator). If a struct has enumerated subtypes, then two additional attributes are generated: * _field_names_: Set of all field names excluding inherited fields. * _fields: List of tuples, where each tuple is (name, validator), and excludes inherited fields. These are needed because serializing a struct with enumerated subtypes requires knowing the fields defined in each level of the hierarchy. """ class_name = class_name_for_data_type(data_type) if data_type.parent_type: parent_type_class_name = class_name_for_data_type( data_type.parent_type, ns) else: parent_type_class_name = None for field in data_type.fields: field_name = fmt_var(field.name) validator_name = generate_validator_constructor(ns, field.data_type) full_validator_name = '{}.{}.validator'.format(class_name, field_name) self.emit('{} = {}'.format(full_validator_name, validator_name)) if field.redactor: self._generate_redactor(full_validator_name, field.redactor) # Generate `_all_field_names_` and `_all_fields_` for every omitted caller (and public). # As an edge case, we union omitted callers with None in the case when the object has no # public fields, as we still need to generate public attributes (`_field_names_` etc) child_omitted_callers = data_type.get_all_omitted_callers() | {None} parent_omitted_callers = data_type.parent_type.get_all_omitted_callers() if \ data_type.parent_type else set([]) for omitted_caller in sorted(child_omitted_callers | parent_omitted_callers, key=str): is_public = omitted_caller is None map_name_prefix = '' if is_public else '_{}'.format(omitted_caller) caller_in_parent = data_type.parent_type and (is_public or omitted_caller in parent_omitted_callers) # generate `_all_field_names_` names_map_name = '{}_field_names_'.format(map_name_prefix) all_names_map_name = '_all{}_field_names_'.format(map_name_prefix) if data_type.is_member_of_enumerated_subtypes_tree(): if is_public or omitted_caller in child_omitted_callers: self.generate_multiline_list( [ "'%s'" % field.name for field in data_type.fields if field.omitted_caller == omitted_caller ], before='{}.{} = set('.format(class_name, names_map_name), after=')', delim=('[', ']'), compact=False) if caller_in_parent: self.emit('{0}.{3} = {1}.{3}.union({0}.{2})' .format(class_name, parent_type_class_name, names_map_name, all_names_map_name)) else: self.emit('{0}.{2} = {0}.{1}'.format(class_name, names_map_name, all_names_map_name)) else: if caller_in_parent: before = '{0}.{1} = {2}.{1}.union(set('.format(class_name, all_names_map_name, parent_type_class_name) after = '))' else: before = '{}.{} = set('.format(class_name, all_names_map_name) after = ')' items = [ "'%s'" % field.name for field in data_type.fields if field.omitted_caller == omitted_caller ] self.generate_multiline_list( items, before=before, after=after, delim=('[', ']'), compact=False) # generate `_all_fields_` fields_map_name = '{}_fields_'.format(map_name_prefix) all_fields_map_name = '_all{}_fields_'.format(map_name_prefix) if data_type.is_member_of_enumerated_subtypes_tree(): items = [] for field in data_type.fields: if field.omitted_caller != omitted_caller: continue var_name = fmt_var(field.name) validator_name = '{}.{}.validator'.format(class_name, var_name) items.append("('{}', {})".format(var_name, validator_name)) self.generate_multiline_list( items, before='{}.{} = '.format(class_name, fields_map_name), delim=('[', ']'), compact=False, ) if caller_in_parent: self.emit('{0}.{3} = {1}.{3} + {0}.{2}'.format( class_name, parent_type_class_name, fields_map_name, all_fields_map_name)) else: self.emit('{0}.{2} = {0}.{1}'.format( class_name, fields_map_name, all_fields_map_name)) else: if caller_in_parent: before = '{0}.{2} = {1}.{2} + '.format( class_name, parent_type_class_name, all_fields_map_name) else: before = '{}.{} = '.format(class_name, all_fields_map_name) items = [] for field in data_type.fields: if field.omitted_caller != omitted_caller: continue var_name = fmt_var(field.name) validator_name = '{}.{}.validator'.format( class_name, var_name) items.append("('{}', {})".format(var_name, validator_name)) self.generate_multiline_list( items, before=before, delim=('[', ']'), compact=False) self.emit() def _generate_struct_attributes_defaults(self, ns, data_type): # Default values can cross-reference, so we also set them after classes. class_name = class_name_for_data_type(data_type) for field in data_type.fields: if field.has_default: self.emit( "{}.{}.default = {}".format( class_name, fmt_var(field.name), self._generate_python_value(ns, field.default)) ) def _generate_struct_class_init(self, data_type): """ Generates constructor. The constructor takes all possible fields as optional arguments. Any argument that is set on construction sets the corresponding field for the instance. """ args = ['self'] for field in data_type.all_fields: field_name_reserved_check = fmt_var(field.name, True) args.append('%s=None' % field_name_reserved_check) self.generate_multiline_list(args, before='def __init__', after=':') with self.indent(): lineno = self.lineno # Call the parent constructor if a super type exists if data_type.parent_type: class_name = class_name_for_data_type(data_type) all_parent_fields = [fmt_func(f.name, check_reserved=True) for f in data_type.parent_type.all_fields] self.generate_multiline_list( all_parent_fields, before='super({}, self).__init__'.format(class_name)) # initialize each field for field in data_type.fields: field_var_name = fmt_var(field.name) self.emit('self._{}_value = bb.NOT_SET'.format(field_var_name)) # handle arguments that were set for field in data_type.fields: field_var_name = fmt_var(field.name, True) self.emit('if {} is not None:'.format(field_var_name)) with self.indent(): self.emit('self.{0} = {0}'.format(field_var_name)) if lineno == self.lineno: self.emit('pass') self.emit() def _generate_python_value(self, ns, value): if is_tag_ref(value): ref = '{}.{}'.format(class_name_for_data_type(value.union_data_type), fmt_var(value.tag_name)) if ns != value.union_data_type.namespace: ref = '{}.{}'.format(fmt_namespace(value.union_data_type.namespace.name), ref) return ref else: return fmt_obj(value) def _generate_struct_class_properties(self, ns, data_type): """ Each field of the struct has a corresponding setter and getter. The setter validates the value being set. """ for field in data_type.fields: field_name = fmt_func(field.name, check_reserved=True) if is_nullable_type(field.data_type): field_dt = field.data_type.data_type dt_nullable = True else: field_dt = field.data_type dt_nullable = False # generate getter for field args = '"{}"'.format(field_name) if dt_nullable: args += ", nullable=True" if is_user_defined_type(field_dt): args += ", user_defined=True" self.emit( '# Instance attribute type: {} (validator is set below)'.format( self._python_type_mapping(ns, field_dt) ) ) self.emit("{} = bb.Attribute({})".format(field_name, args)) self.emit() def _generate_custom_annotation_instance(self, ns, annotation): """ Generates code to construct an instance of an annotation type object with parameters from the specified annotation. """ annotation_class = class_name_for_annotation_type(annotation.annotation_type, ns) return generate_func_call( annotation_class, kwargs=((fmt_var(k, True), self._generate_python_value(ns, v)) for k, v in annotation.kwargs.items()) ) def _generate_custom_annotation_processors(self, ns, data_type, extra_annotations=()): """ Generates code that will run a custom function 'processor' on every field with a custom annotation, no matter how deep (recursively) it might be located in data_type (incl. in elements of lists or maps). If extra_annotations is passed, it's assumed to be a list of custom annotation applied directly onto data_type (e.g. because it's a field in a struct). Yields pairs of (annotation_type, code) where code is code that evaluates to a function that should be executed with an instance of data_type as the only parameter, and whose return value should replace that instance. """ # annotations applied to members of this type dt, _, _ = unwrap(data_type) if is_struct_type(dt) or is_union_type(dt): annotation_types_seen = set() for _, annotation in dt.recursive_custom_annotations: if annotation.annotation_type not in annotation_types_seen: yield (annotation.annotation_type, generate_func_call( 'bb.make_struct_annotation_processor', args=[class_name_for_annotation_type(annotation.annotation_type, ns), 'processor'] )) annotation_types_seen.add(annotation.annotation_type) elif is_list_type(dt): for annotation_type, recursive_processor in self._generate_custom_annotation_processors( ns, dt.data_type): # every member needs to be replaced---use handwritten processor yield (annotation_type, generate_func_call( 'bb.make_list_annotation_processor', args=[recursive_processor] )) elif is_map_type(dt): for annotation_type, recursive_processor in self._generate_custom_annotation_processors( ns, dt.value_data_type): # every value needs to be replaced---use handwritten processor yield (annotation_type, generate_func_call( 'bb.make_map_value_annotation_processor', args=[recursive_processor] )) # annotations applied directly to this type (through aliases or # passed in from the caller) indirect_annotations = dt.recursive_custom_annotations if is_composite_type(dt) else set() all_annotations = (data_type.recursive_custom_annotations if is_composite_type(data_type) else set()) remaining_annotations = [annotation for _, annotation in all_annotations.difference(indirect_annotations)] for annotation in itertools.chain(remaining_annotations, extra_annotations): yield (annotation.annotation_type, generate_func_call( 'bb.partially_apply', args=['processor', self._generate_custom_annotation_instance(ns, annotation)] )) def _generate_struct_class_custom_annotations(self, ns, data_type): """ The _process_custom_annotations function allows client code to access custom annotations defined in the spec. """ self.emit('def _process_custom_annotations(self, annotation_type, field_path, processor):') with self.indent(), emit_pass_if_nothing_emitted(self): self.emit( ( 'super({}, self)._process_custom_annotations(annotation_type, field_path, ' 'processor)' ).format(class_name_for_data_type(data_type)) ) self.emit() for field in data_type.fields: field_name = fmt_var(field.name, check_reserved=True) for annotation_type, processor in self._generate_custom_annotation_processors( ns, field.data_type, field.custom_annotations): annotation_class = class_name_for_annotation_type(annotation_type, ns) self.emit('if annotation_type is {}:'.format(annotation_class)) with self.indent(): self.emit('self.{} = {}'.format( field_name, generate_func_call( processor, args=[ "'{{}}.{}'.format(field_path)".format(field_name), 'self.{}'.format(field_name), ]) )) self.emit() def _generate_enumerated_subtypes_tag_mapping(self, ns, data_type): """ Generates attributes needed for serializing and deserializing structs with enumerated subtypes. These assignments are made after all the Python class definitions to ensure that all references exist. """ assert data_type.has_enumerated_subtypes() # Generate _tag_to_subtype_ attribute: Map from string type tag to # the validator of the referenced subtype. Used on deserialization # to look up the subtype for a given tag. tag_to_subtype_items = [] for tags, subtype in data_type.get_all_subtypes_with_tags(): tag_to_subtype_items.append("{}: {}".format( tags, generate_validator_constructor(ns, subtype))) self.generate_multiline_list( tag_to_subtype_items, before='{}._tag_to_subtype_ = '.format(data_type.name), delim=('{', '}'), compact=False) # Generate _pytype_to_tag_and_subtype_: Map from Python class to a # tuple of (type tag, subtype). Used on serialization to lookup how a # class should be encoded based on the root struct's enumerated # subtypes. items = [] for tag, subtype in data_type.get_all_subtypes_with_tags(): items.append("{0}: ({1}, {2})".format( fmt_class(subtype.name), tag, generate_validator_constructor(ns, subtype))) self.generate_multiline_list( items, before='{}._pytype_to_tag_and_subtype_ = '.format(data_type.name), delim=('{', '}'), compact=False) # Generate _is_catch_all_ attribute: self.emit('{}._is_catch_all_ = {!r}'.format( data_type.name, data_type.is_catch_all())) self.emit() # # Tagged Union Types # def _generate_union_class(self, ns, data_type): # type: (ApiNamespace, Union) -> None """Defines a Python class that represents a union in Stone.""" self.emit(self._class_declaration_for_type(ns, data_type)) with self.indent(): self.emit('"""') if data_type.doc: self.emit_wrapped_text( self.process_doc(data_type.doc, self._docf)) self.emit() self.emit_wrapped_text( 'This class acts as a tagged union. Only one of the ``is_*`` ' 'methods will return true. To get the associated value of a ' 'tag (if one exists), use the corresponding ``get_*`` method.') if data_type.has_documented_fields(): self.emit() for field in data_type.fields: if not field.doc: continue if is_void_type(field.data_type): ivar_doc = ':ivar {}: {}'.format( fmt_namespaced_var(ns.name, data_type.name, field.name), self.process_doc(field.doc, self._docf)) elif is_user_defined_type(field.data_type): if data_type.namespace.name != ns.name: formatted_var = fmt_namespaced_var(ns.name, data_type.name, field.name) else: formatted_var = '{}.{}'.format(data_type.name, fmt_var(field.name)) ivar_doc = ':ivar {} {}: {}'.format( fmt_class(field.data_type.name), formatted_var, self.process_doc(field.doc, self._docf)) else: ivar_doc = ':ivar {} {}: {}'.format( self._python_type_mapping(ns, field.data_type), fmt_namespaced_var(ns.name, data_type.name, field.name), field.doc) self.emit_wrapped_text(ivar_doc, subsequent_prefix=' ') self.emit('"""') self.emit() self._generate_union_class_vars(data_type) self._generate_union_class_variant_creators(ns, data_type) self._generate_union_class_is_set(data_type) self._generate_union_class_get_helpers(ns, data_type) self._generate_union_class_custom_annotations(ns, data_type) self.emit('{0}_validator = bv.Union({0})'.format( class_name_for_data_type(data_type) )) self.emit() def _generate_union_class_vars(self, data_type): """ Adds a _catch_all_ attribute to each class. Also, adds a placeholder attribute for the construction of union members of void type. """ lineno = self.lineno if data_type.catch_all_field: self.emit("_catch_all = '%s'" % data_type.catch_all_field.name) elif not data_type.parent_type: self.emit('_catch_all = None') # Generate stubs for class variables so that IDEs like PyCharms have an # easier time detecting their existence. for field in data_type.fields: if is_void_type(field.data_type): field_name = fmt_var(field.name) self.emit('# Attribute is overwritten below the class definition') self.emit('{} = None'.format(field_name)) if lineno != self.lineno: self.emit() def _generate_union_class_reflection_attributes(self, ns, data_type): """ Adds a class attribute for each union member assigned to a validator. Also adds an attribute that is a map from tag names to validators. """ class_name = fmt_class(data_type.name) for field in data_type.fields: field_name = fmt_var(field.name) validator_name = generate_validator_constructor( ns, field.data_type) full_validator_name = '{}._{}_validator'.format(class_name, field_name) self.emit('{} = {}'.format(full_validator_name, validator_name)) if field.redactor: self._generate_redactor(full_validator_name, field.redactor) # generate _all_fields_ for each omitted caller (and public) child_omitted_callers = data_type.get_all_omitted_callers() parent_omitted_callers = data_type.parent_type.get_all_omitted_callers() if \ data_type.parent_type else set([]) all_omitted_callers = child_omitted_callers | parent_omitted_callers if len(all_omitted_callers) != 0: self.emit('{}._permissioned_tagmaps = {}'.format(class_name, all_omitted_callers)) for omitted_caller in sorted(all_omitted_callers | {None}, key=str): is_public = omitted_caller is None tagmap_name = '_tagmap' if is_public else '_{}_tagmap'.format(omitted_caller) caller_in_parent = data_type.parent_type and (is_public or omitted_caller in parent_omitted_callers) with self.block('{}.{} ='.format(class_name, tagmap_name)): for field in data_type.fields: if field.omitted_caller != omitted_caller: continue var_name = fmt_var(field.name) validator_name = '{}._{}_validator'.format(class_name, var_name) self.emit("'{}': {},".format(var_name, validator_name)) if caller_in_parent: self.emit('{0}.{1}.update({2}.{1})'.format( class_name, tagmap_name, class_name_for_data_type(data_type.parent_type, ns)) ) self.emit() def _generate_union_class_variant_creators(self, ns, data_type): """ Each non-symbol, non-any variant has a corresponding class method that can be used to construct a union with that variant selected. """ for field in data_type.fields: if not is_void_type(field.data_type): field_name = fmt_func(field.name) field_name_reserved_check = fmt_func(field.name, check_reserved=True) if is_nullable_type(field.data_type): field_dt = field.data_type.data_type else: field_dt = field.data_type self.emit('@classmethod') self.emit('def {}(cls, val):'.format(field_name_reserved_check)) with self.indent(): self.emit('"""') self.emit_wrapped_text( 'Create an instance of this class set to the ``%s`` ' 'tag with value ``val``.' % field_name) self.emit() self.emit(':param {} val:'.format( self._python_type_mapping(ns, field_dt))) self.emit(':rtype: {}'.format( self._python_type_mapping(ns, data_type))) self.emit('"""') self.emit("return cls('{}', val)".format(field_name)) self.emit() def _generate_union_class_is_set(self, data_type): for field in data_type.fields: field_name = fmt_func(field.name) self.emit('def is_{}(self):'.format(field_name)) with self.indent(): self.emit('"""') self.emit('Check if the union tag is ``%s``.' % field_name) self.emit() self.emit(':rtype: bool') self.emit('"""') self.emit("return self._tag == '{}'".format(field_name)) self.emit() def _generate_union_class_get_helpers(self, ns, data_type): """ These are the getters used to access the value of a variant, once the tag has been switched on. """ for field in data_type.fields: field_name = fmt_func(field.name) if not is_void_type(field.data_type): # generate getter for field self.emit('def get_{}(self):'.format(field_name)) with self.indent(): if is_nullable_type(field.data_type): field_dt = field.data_type.data_type else: field_dt = field.data_type self.emit('"""') if field.doc: self.emit_wrapped_text( self.process_doc(field.doc, self._docf)) self.emit() self.emit("Only call this if :meth:`is_%s` is true." % field_name) # Sphinx wants an extra line between the text and the # rtype declaration. self.emit() self.emit(':rtype: {}'.format( self._python_type_mapping(ns, field_dt))) self.emit('"""') self.emit('if not self.is_{}():'.format(field_name)) with self.indent(): self.emit( 'raise AttributeError("tag \'{}\' not set")'.format( field_name)) self.emit('return self._value') self.emit() def _generate_union_class_custom_annotations(self, ns, data_type): """ The _process_custom_annotations function allows client code to access custom annotations defined in the spec. """ self.emit('def _process_custom_annotations(self, annotation_type, field_path, processor):') with self.indent(), emit_pass_if_nothing_emitted(self): self.emit( ( 'super({}, self)._process_custom_annotations(annotation_type, field_path, ' 'processor)' ).format(class_name_for_data_type(data_type)) ) self.emit() for field in data_type.fields: recursive_processors = list(self._generate_custom_annotation_processors( ns, field.data_type, field.custom_annotations)) # check if we have any annotations that apply to this field at all if len(recursive_processors) == 0: continue field_name = fmt_func(field.name) self.emit('if self.is_{}():'.format(field_name)) with self.indent(): for annotation_type, processor in recursive_processors: annotation_class = class_name_for_annotation_type(annotation_type, ns) self.emit('if annotation_type is {}:'.format(annotation_class)) with self.indent(): self.emit('self._value = {}'.format( generate_func_call( processor, args=[ "'{{}}.{}'.format(field_path)".format(field_name), 'self._value', ]) )) self.emit() def _generate_union_class_symbol_creators(self, data_type): """ Class attributes that represent a symbol are set after the union class definition. """ class_name = fmt_class(data_type.name) lineno = self.lineno for field in data_type.fields: if is_void_type(field.data_type): field_name = fmt_func(field.name) self.emit("{0}.{1} = {0}('{1}')".format(class_name, field_name)) if lineno != self.lineno: self.emit() def _generate_routes(self, route_schema, namespace): check_route_name_conflict(namespace) for route in namespace.routes: data_types = [route.arg_data_type, route.result_data_type, route.error_data_type] with self.block( '{} = bb.Route('.format(fmt_func(route.name, version=route.version)), delim=(None, None), after=')'): self.emit("'{}',".format(route.name)) self.emit('{},'.format(route.version)) self.emit('{!r},'.format(route.deprecated is not None)) for data_type in data_types: self.emit( generate_validator_constructor(namespace, data_type) + ',') attrs = [] for field in route_schema.fields: attr_key = field.name attrs.append("'{}': {!r}".format(attr_key, route.attrs.get(attr_key))) self.generate_multiline_list( attrs, delim=('{', '}'), after=',', compact=True) if namespace.routes: self.emit() with self.block('ROUTES =', delim=('{', '}')): for route in namespace.routes: self.emit("'{}': {},".format( route.name_with_version(), fmt_func(route.name, version=route.version))) self.emit() def _generate_redactor(self, validator_name, redactor): regex = "'{}'".format(redactor.regex) if redactor.regex else 'None' if isinstance(redactor, RedactedHash): self.emit("{}._redact = bv.HashRedactor({})".format(validator_name, regex)) elif isinstance(redactor, RedactedBlot): self.emit("{}._redact = bv.BlotRedactor({})".format(validator_name, regex)) def generate_validator_constructor(ns, data_type): """ Given a Stone data type, returns a string that can be used to construct the appropriate validation object in Python. """ dt, nullable_dt = unwrap_nullable(data_type) if is_list_type(dt): v = generate_func_call( 'bv.List', args=[ generate_validator_constructor(ns, dt.data_type)], kwargs=[ ('min_items', dt.min_items), ('max_items', dt.max_items)], ) elif is_map_type(dt): v = generate_func_call( 'bv.Map', args=[ generate_validator_constructor(ns, dt.key_data_type), generate_validator_constructor(ns, dt.value_data_type), ] ) elif is_numeric_type(dt): v = generate_func_call( 'bv.{}'.format(dt.name), kwargs=[ ('min_value', dt.min_value), ('max_value', dt.max_value)], ) elif is_string_type(dt): pattern = None if dt.pattern is not None: pattern = repr(dt.pattern) v = generate_func_call( 'bv.String', kwargs=[ ('min_length', dt.min_length), ('max_length', dt.max_length), ('pattern', pattern)], ) elif is_timestamp_type(dt): v = generate_func_call( 'bv.Timestamp', args=[repr(dt.format)], ) elif is_user_defined_type(dt): v = fmt_class(dt.name) + '_validator' if ns.name != dt.namespace.name: v = '{}.{}'.format(fmt_namespace(dt.namespace.name), v) elif is_alias(dt): # Assume that the alias has already been declared elsewhere. name = fmt_class(dt.name) + '_validator' if ns.name != dt.namespace.name: name = '{}.{}'.format(fmt_namespace(dt.namespace.name), name) v = name elif is_boolean_type(dt) or is_bytes_type(dt) or is_void_type(dt): v = generate_func_call('bv.{}'.format(dt.name)) else: raise AssertionError('Unsupported data type: %r' % dt) if nullable_dt: return generate_func_call('bv.Nullable', args=[v]) else: return v def generate_func_call(name, args=None, kwargs=None): """ Generates code to call a function. Args: name (str): The function name. args (list[str]): Each positional argument. kwargs (list[tuple]): Each tuple is (arg: str, value: str). If value is None, then the keyword argument is omitted. Otherwise, if the value is not a string, then str() is called on it. Returns: str: Code to call a function. """ all_args = [] if args: all_args.extend(args) if kwargs: all_args.extend('{}={}'.format(k, v) for k, v in kwargs if v is not None) return '{}({})'.format(name, ', '.join(all_args)) stone-3.3.1/stone/backends/swift.py000066400000000000000000000130761417406541500172740ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from contextlib import contextmanager from stone.ir import ( Boolean, Bytes, DataType, Float32, Float64, Int32, Int64, List, String, Timestamp, UInt32, UInt64, Void, is_list_type, is_timestamp_type, is_union_type, is_user_defined_type, unwrap_nullable, ) from stone.backend import CodeBackend from stone.backends.swift_helpers import ( fmt_class, fmt_func, fmt_obj, fmt_type, fmt_var, ) _serial_type_table = { Boolean: 'BoolSerializer', Bytes: 'NSDataSerializer', Float32: 'FloatSerializer', Float64: 'DoubleSerializer', Int32: 'Int32Serializer', Int64: 'Int64Serializer', List: 'ArraySerializer', String: 'StringSerializer', Timestamp: 'NSDateSerializer', UInt32: 'UInt32Serializer', UInt64: 'UInt64Serializer', Void: 'VoidSerializer', } stone_warning = """\ /// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// /// Auto-generated by Stone, do not modify. /// """ # This will be at the top of the generated file. base = """\ {}\ import Foundation """.format(stone_warning) undocumented = '(no description)' class SwiftBaseBackend(CodeBackend): """Wrapper class over Stone generator for Swift logic.""" # pylint: disable=abstract-method @contextmanager def function_block(self, func, args, return_type=None): signature = '{}({})'.format(func, args) if return_type: signature += ' -> {}'.format(return_type) with self.block(signature): yield def _func_args(self, args_list, newlines=False, force_first=False, not_init=False): out = [] first = True for k, v in args_list: # this is a temporary hack -- injected client-side args # do not have a separate field for default value. Right now, # default values are stored along with the type, e.g. # `Bool = True` is a type, hence this check. if first and force_first and '=' not in v: k = "{0} {0}".format(k) if first and v is not None and not_init: out.append('{}'.format(v)) elif v is not None: out.append('{}: {}'.format(k, v)) first = False sep = ', ' if newlines: sep += '\n' + self.make_indent() return sep.join(out) @contextmanager def class_block(self, thing, protocols=None): protocols = protocols or [] extensions = [] if isinstance(thing, DataType): name = fmt_class(thing.name) if thing.parent_type: extensions.append(fmt_type(thing.parent_type)) else: name = thing extensions.extend(protocols) extend_suffix = ': {}'.format(', '.join(extensions)) if extensions else '' with self.block('open class {}{}'.format(name, extend_suffix)): yield def _struct_init_args(self, data_type, namespace=None): # pylint: disable=unused-argument args = [] for field in data_type.all_fields: name = fmt_var(field.name) value = fmt_type(field.data_type) data_type, nullable = unwrap_nullable(field.data_type) if field.has_default: if is_union_type(data_type): default = '.{}'.format(fmt_var(field.default.tag_name)) else: default = fmt_obj(field.default) value += ' = {}'.format(default) elif nullable: value += ' = nil' arg = (name, value) args.append(arg) return args def _docf(self, tag, val): if tag == 'route': if ':' in val: val, version = val.split(':', 1) version = int(version) else: version = 1 return fmt_func(val, version) elif tag == 'field': if '.' in val: cls, field = val.split('.') return ('{} in {}'.format(fmt_var(field), fmt_class(cls))) else: return fmt_var(val) elif tag in ('type', 'val', 'link'): return val else: return val def fmt_serial_type(data_type): data_type, nullable = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = '{}.{}Serializer' result = result.format(fmt_class(data_type.namespace.name), fmt_class(data_type.name)) else: result = _serial_type_table.get(data_type.__class__, fmt_class(data_type.name)) if is_list_type(data_type): result = result + '<{}>'.format(fmt_serial_type(data_type.data_type)) return result if not nullable else 'NullableSerializer' def fmt_serial_obj(data_type): data_type, nullable = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = '{}.{}Serializer()' result = result.format(fmt_class(data_type.namespace.name), fmt_class(data_type.name)) else: result = _serial_type_table.get(data_type.__class__, fmt_class(data_type.name)) if is_list_type(data_type): result = result + '({})'.format(fmt_serial_obj(data_type.data_type)) elif is_timestamp_type(data_type): result = result + '("{}")'.format(data_type.format) else: result = 'Serialization._{}'.format(result) return result if not nullable else 'NullableSerializer({})'.format(result) stone-3.3.1/stone/backends/swift_client.py000066400000000000000000000251721417406541500206320ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json from stone.ir import ( is_struct_type, is_union_type, is_void_type, ) from stone.backends.swift import ( base, fmt_serial_type, SwiftBaseBackend, undocumented, ) from stone.backends.swift_helpers import ( check_route_name_conflict, fmt_class, fmt_func, fmt_var, fmt_type, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any _cmdline_parser = argparse.ArgumentParser( prog='swift-client-backend', description=( 'Generates a Swift class with an object for each namespace, and in each ' 'namespace object, a method for each route. This class assumes that the ' 'swift_types backend was used with the same output directory.'), ) _cmdline_parser.add_argument( '-m', '--module-name', required=True, type=str, help=('The name of the Swift module to generate. Please exclude the .swift ' 'file extension.'), ) _cmdline_parser.add_argument( '-c', '--class-name', required=True, type=str, help=('The name of the Swift class that contains an object for each namespace, ' 'and in each namespace object, a method for each route.') ) _cmdline_parser.add_argument( '-t', '--transport-client-name', required=True, type=str, help='The name of the Swift class that manages network API calls.', ) _cmdline_parser.add_argument( '-y', '--client-args', required=True, type=str, help='The client-side route arguments to append to each route by style type.', ) _cmdline_parser.add_argument( '-z', '--style-to-request', required=True, type=str, help='The dict that maps a style type to a Swift request object name.', ) class SwiftBackend(SwiftBaseBackend): """ Generates Swift client base that implements route interfaces. Examples: ``` open class ExampleClientBase { /// Routes within the namespace1 namespace. See Namespace1 for details. open var namespace1: Namespace1! /// Routes within the namespace2 namespace. See Namespace2 for details. open var namespace2: Namespace2! public init(client: ExampleTransportClient) { self.namespace1 = Namespace1(client: client) self.namespace2 = Namespace2(client: client) } } ``` Here, `ExampleTransportClient` would contain the implementation of a handwritten, project-specific networking client. Additionally, the `Namespace1` object would have as its methods all routes in the `Namespace1` namespace. A hypothetical 'copy' enpoding might be implemented like: ``` open func copy(fromPath fromPath: String, toPath: String) -> ExampleRequestType { let route = Namespace1.copy let serverArgs = Namespace1.CopyArg(fromPath: fromPath, toPath: toPath) return client.request(route, serverArgs: serverArgs) } ``` Here, ExampleRequestType is a project-specific request type, parameterized by response and error serializers. """ cmdline_parser = _cmdline_parser def generate(self, api): for namespace in api.namespaces.values(): ns_class = fmt_class(namespace.name) if namespace.routes: with self.output_to_relative_path('{}Routes.swift'.format(ns_class)): self._generate_routes(namespace) with self.output_to_relative_path('{}.swift'.format(self.args.module_name)): self._generate_client(api) def _generate_client(self, api): self.emit_raw(base) self.emit('import Alamofire') self.emit() with self.block('open class {}'.format(self.args.class_name)): namespace_fields = [] for namespace in api.namespaces.values(): if namespace.routes: namespace_fields.append((namespace.name, fmt_class(namespace.name))) for var, typ in namespace_fields: self.emit('/// Routes within the {} namespace. ' 'See {}Routes for details.'.format(var, typ)) self.emit('open var {}: {}Routes!'.format(var, typ)) self.emit() with self.function_block('public init', args=self._func_args( [('client', '{}'.format(self.args.transport_client_name))])): for var, typ in namespace_fields: self.emit('self.{} = {}Routes(client: client)'.format(var, typ)) def _generate_routes(self, namespace): check_route_name_conflict(namespace) ns_class = fmt_class(namespace.name) self.emit_raw(base) self.emit('/// Routes for the {} namespace'.format(namespace.name)) with self.block('open class {}Routes'.format(ns_class)): self.emit('public let client: {}'.format(self.args.transport_client_name)) args = [('client', '{}'.format(self.args.transport_client_name))] with self.function_block('init', self._func_args(args)): self.emit('self.client = client') self.emit() for route in namespace.routes: self._generate_route(namespace, route) def _get_route_args(self, namespace, route): data_type = route.arg_data_type arg_type = fmt_type(data_type) if is_struct_type(data_type): arg_list = self._struct_init_args(data_type, namespace=namespace) doc_list = [(fmt_var(f.name), self.process_doc(f.doc, self._docf) if f.doc else undocumented) for f in data_type.fields if f.doc] elif is_union_type(data_type): arg_list = [(fmt_var(data_type.name), '{}.{}'.format( fmt_class(namespace.name), fmt_class(data_type.name)))] doc_list = [(fmt_var(data_type.name), self.process_doc(data_type.doc, self._docf) if data_type.doc else 'The {} union'.format(fmt_class(data_type.name)))] else: arg_list = [] if is_void_type(data_type) else [('request', arg_type)] doc_list = [] return arg_list, doc_list def _emit_route(self, namespace, route, req_obj_name, extra_args=None, extra_docs=None): arg_list, doc_list = self._get_route_args(namespace, route) extra_args = extra_args or [] extra_docs = extra_docs or [] arg_type = fmt_type(route.arg_data_type) func_name = fmt_func(route.name, route.version) if route.doc: route_doc = self.process_doc(route.doc, self._docf) else: route_doc = 'The {} route'.format(func_name) self.emit_wrapped_text(route_doc, prefix='/// ', width=120) self.emit('///') for name, doc in doc_list + extra_docs: param_doc = '- parameter {}: {}'.format(name, doc if doc is not None else undocumented) self.emit_wrapped_text(param_doc, prefix='/// ', width=120) self.emit('///') output = (' - returns: Through the response callback, the caller will ' + 'receive a `{}` object on success or a `{}` object on failure.') output = output.format(fmt_type(route.result_data_type), fmt_type(route.error_data_type)) self.emit_wrapped_text(output, prefix='/// ', width=120) func_args = [ ('route', '{}.{}'.format(fmt_class(namespace.name), func_name)), ] client_args = [] return_args = [('route', 'route')] for name, value, typ in extra_args: arg_list.append((name, typ)) func_args.append((name, value)) client_args.append((name, value)) rtype = fmt_serial_type(route.result_data_type) etype = fmt_serial_type(route.error_data_type) self._maybe_generate_deprecation_warning(route) with self.function_block('@discardableResult open func {}'.format(func_name), args=self._func_args(arg_list, force_first=False), return_type='{}<{}, {}>'.format(req_obj_name, rtype, etype)): self.emit('let route = {}.{}'.format(fmt_class(namespace.name), func_name)) if is_struct_type(route.arg_data_type): args = [(name, name) for name, _ in self._struct_init_args(route.arg_data_type)] func_args += [('serverArgs', '{}({})'.format(arg_type, self._func_args(args)))] self.emit('let serverArgs = {}({})'.format(arg_type, self._func_args(args))) elif is_union_type(route.arg_data_type): self.emit('let serverArgs = {}'.format(fmt_var(route.arg_data_type.name))) if not is_void_type(route.arg_data_type): return_args += [('serverArgs', 'serverArgs')] return_args += client_args txt = 'return client.request({})'.format( self._func_args(return_args, not_init=True) ) self.emit(txt) self.emit() def _maybe_generate_deprecation_warning(self, route): if route.deprecated: msg = '{} is deprecated.'.format(fmt_func(route.name, route.version)) if route.deprecated.by: msg += ' Use {}.'.format( fmt_func(route.deprecated.by.name, route.deprecated.by.version)) self.emit('@available(*, unavailable, message:"{}")'.format(msg)) def _generate_route(self, namespace, route): route_type = route.attrs.get('style') client_args = json.loads(self.args.client_args) style_to_request = json.loads(self.args.style_to_request) if route_type not in client_args.keys(): self._emit_route(namespace, route, style_to_request[route_type]) else: for args_data in client_args[route_type]: req_obj_key, type_data_list = tuple(args_data) req_obj_name = style_to_request[req_obj_key] extra_args = [tuple(type_data[:-1]) for type_data in type_data_list] extra_docs = [(type_data[0], type_data[-1]) for type_data in type_data_list] self._emit_route(namespace, route, req_obj_name, extra_args, extra_docs) stone-3.3.1/stone/backends/swift_helpers.py000066400000000000000000000100161417406541500210050ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import pprint from stone.ir import ( Boolean, Bytes, Float32, Float64, Int32, Int64, List, String, Timestamp, UInt32, UInt64, Void, is_boolean_type, is_list_type, is_numeric_type, is_string_type, is_tag_ref, is_user_defined_type, unwrap_nullable, ) from .helpers import split_words # This file defines *stylistic* choices for Swift # (ie, that class names are UpperCamelCase and that variables are lowerCamelCase) _type_table = { Boolean: 'Bool', Bytes: 'Data', Float32: 'Float', Float64: 'Double', Int32: 'Int32', Int64: 'Int64', List: 'Array', String: 'String', Timestamp: 'Date', UInt32: 'UInt32', UInt64: 'UInt64', Void: 'Void', } _reserved_words = { 'description', 'bool', 'nsdata' 'float', 'double', 'int32', 'int64', 'list', 'string', 'timestamp', 'uint32', 'uint64', 'void', 'associatedtype', 'class', 'deinit', 'enum', 'extension', 'func', 'import', 'init', 'inout', 'internal', 'let', 'operator', 'private', 'protocol', 'public', 'static', 'struct', 'subscript', 'typealias', 'var', 'default', } def fmt_obj(o): assert not isinstance(o, dict), "Only use for base type literals" if o is True: return 'true' if o is False: return 'false' if o is None: return 'nil' if o == '': return '""' return pprint.pformat(o, width=1) def _format_camelcase(name, lower_first=True): words = [word.capitalize() for word in split_words(name)] if lower_first: words[0] = words[0].lower() ret = ''.join(words) if ret.lower() in _reserved_words: ret += '_' return ret def fmt_class(name): return _format_camelcase(name, lower_first=False) def fmt_func(name, version): if version > 1: name = '{}_v{}'.format(name, version) name = _format_camelcase(name) return name def fmt_type(data_type): data_type, nullable = unwrap_nullable(data_type) if is_user_defined_type(data_type): result = '{}.{}'.format(fmt_class(data_type.namespace.name), fmt_class(data_type.name)) else: result = _type_table.get(data_type.__class__, fmt_class(data_type.name)) if is_list_type(data_type): result = result + '<{}>'.format(fmt_type(data_type.data_type)) return result if not nullable else result + '?' def fmt_var(name): return _format_camelcase(name) def fmt_default_value(namespace, field): if is_tag_ref(field.default): return '{}.{}Serializer().serialize(.{})'.format( fmt_class(namespace.name), fmt_class(field.default.union_data_type.name), fmt_var(field.default.tag_name)) elif is_list_type(field.data_type): return '.array({})'.format(field.default) elif is_numeric_type(field.data_type): return '.number({})'.format(field.default) elif is_string_type(field.data_type): return '.str({})'.format(fmt_obj(field.default)) elif is_boolean_type(field.data_type): if field.default: bool_str = '1' else: bool_str = '0' return '.number({})'.format(bool_str) else: raise TypeError('Can\'t handle default value type %r' % type(field.data_type)) def check_route_name_conflict(namespace): """ Check name conflicts among generated route definitions. Raise a runtime exception when a conflict is encountered. """ route_by_name = {} for route in namespace.routes: route_name = fmt_func(route.name, route.version) if route_name in route_by_name: other_route = route_by_name[route_name] raise RuntimeError( 'There is a name conflict between {!r} and {!r}'.format(other_route, route)) route_by_name[route_name] = route stone-3.3.1/stone/backends/swift_rsrc/000077500000000000000000000000001417406541500177445ustar00rootroot00000000000000stone-3.3.1/stone/backends/swift_rsrc/StoneBase.swift000066400000000000000000000021051417406541500227030ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// import Foundation // The objects in this file are used by generated code and should not need to be invoked manually. open class Route { public let name: String public let version: Int32 public let namespace: String public let deprecated: Bool public let argSerializer: ASerial public let responseSerializer: RSerial public let errorSerializer: ESerial public let attrs: [String: String?] public init(name: String, version: Int32, namespace: String, deprecated: Bool, argSerializer: ASerial, responseSerializer: RSerial, errorSerializer: ESerial, attrs: [String: String?]) { self.name = name self.version = version self.namespace = namespace self.deprecated = deprecated self.argSerializer = argSerializer self.responseSerializer = responseSerializer self.errorSerializer = errorSerializer self.attrs = attrs } } stone-3.3.1/stone/backends/swift_rsrc/StoneSerializers.swift000066400000000000000000000316471417406541500243420ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// import Foundation // The objects in this file are used by generated code and should not need to be invoked manually. public enum JSON { case array([JSON]) case dictionary([String: JSON]) case str(String) case number(NSNumber) case null } open class SerializeUtil { open class func objectToJSON(_ json: AnyObject) -> JSON { switch json { case _ as NSNull: return .null case let num as NSNumber: return .number(num) case let str as String: return .str(str) case let dict as [String: AnyObject]: var ret = [String: JSON]() for (k, v) in dict { ret[k] = objectToJSON(v) } return .dictionary(ret) case let array as [AnyObject]: return .array(array.map(objectToJSON)) default: fatalError("Unknown type trying to parse JSON.") } } open class func prepareJSONForSerialization(_ json: JSON) -> AnyObject { switch json { case .array(let array): return array.map(prepareJSONForSerialization) as AnyObject case .dictionary(let dict): var ret = [String: AnyObject]() for (k, v) in dict { // kind of a hack... switch v { case .null: continue default: ret[k] = prepareJSONForSerialization(v) } } return ret as AnyObject case .number(let n): return n case .str(let s): return s as AnyObject case .null: return NSNull() } } open class func dumpJSON(_ json: JSON) -> Data? { switch json { case .null: return "null".data(using: String.Encoding.utf8, allowLossyConversion: false) default: let obj: AnyObject = prepareJSONForSerialization(json) if JSONSerialization.isValidJSONObject(obj) { return try! JSONSerialization.data(withJSONObject: obj, options: JSONSerialization.WritingOptions()) } else { fatalError("Invalid JSON toplevel type") } } } open class func parseJSON(_ data: Data) -> JSON { let obj: AnyObject = try! JSONSerialization.jsonObject(with: data, options: JSONSerialization.ReadingOptions.allowFragments) as AnyObject return objectToJSON(obj) } } public protocol JSONSerializer { associatedtype ValueType func serialize(_: ValueType) -> JSON func deserialize(_: JSON) -> ValueType } open class VoidSerializer: JSONSerializer { open func serialize(_ value: Void) -> JSON { return .null } open func deserialize(_ json: JSON) -> Void { switch json { case .null: return default: fatalError("Type error deserializing") } } } open class ArraySerializer: JSONSerializer { var elementSerializer: T init(_ elementSerializer: T) { self.elementSerializer = elementSerializer } open func serialize(_ arr: Array) -> JSON { return .array(arr.map { self.elementSerializer.serialize($0) }) } open func deserialize(_ json: JSON) -> Array { switch json { case .array(let arr): return arr.map { self.elementSerializer.deserialize($0) } default: fatalError("Type error deserializing") } } } open class StringSerializer: JSONSerializer { open func serialize(_ value: String) -> JSON { return .str(value) } open func deserialize(_ json: JSON) -> String { switch (json) { case .str(let s): return s default: fatalError("Type error deserializing") } } } open class NSDateSerializer: JSONSerializer { var dateFormatter: DateFormatter fileprivate func convertFormat(_ format: String) -> String? { func symbolForToken(_ token: String) -> String { switch token { case "%a": // Weekday as locale’s abbreviated name. return "EEE" case "%A": // Weekday as locale’s full name. return "EEEE" case "%w": // Weekday as a decimal number, where 0 is Sunday and 6 is Saturday. 0, 1, ..., 6 return "ccccc" case "%d": // Day of the month as a zero-padded decimal number. 01, 02, ..., 31 return "dd" case "%b": // Month as locale’s abbreviated name. return "MMM" case "%B": // Month as locale’s full name. return "MMMM" case "%m": // Month as a zero-padded decimal number. 01, 02, ..., 12 return "MM" case "%y": // Year without century as a zero-padded decimal number. 00, 01, ..., 99 return "yy" case "%Y": // Year with century as a decimal number. 1970, 1988, 2001, 2013 return "yyyy" case "%H": // Hour (24-hour clock) as a zero-padded decimal number. 00, 01, ..., 23 return "HH" case "%I": // Hour (12-hour clock) as a zero-padded decimal number. 01, 02, ..., 12 return "hh" case "%p": // Locale’s equivalent of either AM or PM. return "a" case "%M": // Minute as a zero-padded decimal number. 00, 01, ..., 59 return "mm" case "%S": // Second as a zero-padded decimal number. 00, 01, ..., 59 return "ss" case "%f": // Microsecond as a decimal number, zero-padded on the left. 000000, 000001, ..., 999999 return "SSSSSS" case "%z": // UTC offset in the form +HHMM or -HHMM (empty string if the the object is naive). (empty), +0000, -0400, +1030 return "Z" case "%Z": // Time zone name (empty string if the object is naive). (empty), UTC, EST, CST return "z" case "%j": // Day of the year as a zero-padded decimal number. 001, 002, ..., 366 return "DDD" case "%U": // Week number of the year (Sunday as the first day of the week) as a zero padded decimal number. All days in a new year preceding the first Sunday are considered to be in week 0. 00, 01, ..., 53 (6) return "ww" case "%W": // Week number of the year (Monday as the first day of the week) as a decimal number. All days in a new year preceding the first Monday are considered to be in week 0. 00, 01, ..., 53 (6) return "ww" // one of these can't be right case "%c": // Locale’s appropriate date and time representation. return "" // unsupported case "%x": // Locale’s appropriate date representation. return "" // unsupported case "%X": // Locale’s appropriate time representation. return "" // unsupported case "%%": // A literal '%' character. return "%" default: return "" } } var newFormat = "" var inQuotedText = false var i = format.startIndex while i < format.endIndex { if format[i] == "%" { if format.index(after: i) >= format.endIndex { return nil } i = format.index(after: i) let token = "%\(format[i])" if inQuotedText { newFormat += "'" inQuotedText = false } newFormat += symbolForToken(token) } else { if "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ".contains(format[i]) { if !inQuotedText { newFormat += "'" inQuotedText = true } } else if format[i] == "'" { newFormat += "'" } newFormat += String(format[i]) } i = format.index(after: i) } if inQuotedText { newFormat += "'" } return newFormat } init(_ dateFormat: String) { self.dateFormatter = DateFormatter() self.dateFormatter.timeZone = TimeZone(identifier: "UTC") self.dateFormatter.locale = Locale(identifier: "en_US_POSIX") dateFormatter.dateFormat = self.convertFormat(dateFormat) } open func serialize(_ value: Date) -> JSON { return .str(self.dateFormatter.string(from: value)) } open func deserialize(_ json: JSON) -> Date { switch json { case .str(let s): return self.dateFormatter.date(from: s)! default: fatalError("Type error deserializing") } } } open class BoolSerializer: JSONSerializer { open func serialize(_ value: Bool) -> JSON { return .number(NSNumber(value: value as Bool)) } open func deserialize(_ json: JSON) -> Bool { switch json { case .number(let b): return b.boolValue default: fatalError("Type error deserializing") } } } open class UInt64Serializer: JSONSerializer { open func serialize(_ value: UInt64) -> JSON { return .number(NSNumber(value: value as UInt64)) } open func deserialize(_ json: JSON) -> UInt64 { switch json { case .number(let n): return n.uint64Value default: fatalError("Type error deserializing") } } } open class Int64Serializer: JSONSerializer { open func serialize(_ value: Int64) -> JSON { return .number(NSNumber(value: value as Int64)) } open func deserialize(_ json: JSON) -> Int64 { switch json { case .number(let n): return n.int64Value default: fatalError("Type error deserializing") } } } open class Int32Serializer: JSONSerializer { open func serialize(_ value: Int32) -> JSON { return .number(NSNumber(value: value as Int32)) } open func deserialize(_ json: JSON) -> Int32 { switch json { case .number(let n): return n.int32Value default: fatalError("Type error deserializing") } } } open class UInt32Serializer: JSONSerializer { open func serialize(_ value: UInt32) -> JSON { return .number(NSNumber(value: value as UInt32)) } open func deserialize(_ json: JSON) -> UInt32 { switch json { case .number(let n): return n.uint32Value default: fatalError("Type error deserializing") } } } open class NSDataSerializer: JSONSerializer { open func serialize(_ value: Data) -> JSON { return .str(value.base64EncodedString(options: [])) } open func deserialize(_ json: JSON) -> Data { switch(json) { case .str(let s): return Data(base64Encoded: s, options: [])! default: fatalError("Type error deserializing") } } } open class DoubleSerializer: JSONSerializer { open func serialize(_ value: Double) -> JSON { return .number(NSNumber(value: value as Double)) } open func deserialize(_ json: JSON) -> Double { switch json { case .number(let n): return n.doubleValue default: fatalError("Type error deserializing") } } } open class NullableSerializer: JSONSerializer { var internalSerializer: T init(_ serializer: T) { self.internalSerializer = serializer } open func serialize(_ value: Optional) -> JSON { if let v = value { return internalSerializer.serialize(v) } else { return .null } } open func deserialize(_ json: JSON) -> Optional { switch json { case .null: return nil default: return internalSerializer.deserialize(json) } } } struct Serialization { static var _StringSerializer = StringSerializer() static var _BoolSerializer = BoolSerializer() static var _UInt64Serializer = UInt64Serializer() static var _UInt32Serializer = UInt32Serializer() static var _Int64Serializer = Int64Serializer() static var _Int32Serializer = Int32Serializer() static var _VoidSerializer = VoidSerializer() static var _NSDataSerializer = NSDataSerializer() static var _DoubleSerializer = DoubleSerializer() static func getFields(_ json: JSON) -> [String: JSON] { switch json { case .dictionary(let dict): return dict default: fatalError("Type error") } } static func getTag(_ d: [String: JSON]) -> String { return _StringSerializer.deserialize(d[".tag"]!) } } stone-3.3.1/stone/backends/swift_rsrc/StoneValidators.swift000066400000000000000000000056231417406541500241510ustar00rootroot00000000000000/// /// Copyright (c) 2016 Dropbox, Inc. All rights reserved. /// import Foundation // The objects in this file are used by generated code and should not need to be invoked manually. var _assertFunc: (Bool, String) -> Void = { cond, message in precondition(cond, message) } public func setAssertFunc(_ assertFunc: @escaping (Bool, String) -> Void) { _assertFunc = assertFunc } public func arrayValidator(minItems: Int? = nil, maxItems: Int? = nil, itemValidator: @escaping (T) -> Void) -> (Array) -> Void { return { (value: Array) -> Void in if let minItems = minItems { _assertFunc(value.count >= minItems, "\(value) must have at least \(minItems) items") } if let maxItems = maxItems { _assertFunc(value.count <= maxItems, "\(value) must have at most \(maxItems) items") } for el in value { itemValidator(el) } } } public func stringValidator(minLength: Int? = nil, maxLength: Int? = nil, pattern: String? = nil) -> (String) -> Void { return { (value: String) -> Void in let length = value.count if let minLength = minLength { _assertFunc(length >= minLength, "\"\(value)\" must be at least \(minLength) characters") } if let maxLength = maxLength { _assertFunc(length <= maxLength, "\"\(value)\" must be at most \(maxLength) characters") } if let pat = pattern { // patterns much match entire input sequence let re = try! NSRegularExpression(pattern: "\\A(?:\(pat))\\z", options: NSRegularExpression.Options()) let matches = re.matches(in: value, options: NSRegularExpression.MatchingOptions(), range: NSRange(location: 0, length: length)) _assertFunc(matches.count > 0, "\"\(value) must match pattern \"\(re.pattern)\"") } } } public func comparableValidator(minValue: T? = nil, maxValue: T? = nil) -> (T) -> Void { return { (value: T) -> Void in if let minValue = minValue { _assertFunc(minValue <= value, "\(value) must be at least \(minValue)") } if let maxValue = maxValue { _assertFunc(maxValue >= value, "\(value) must be at most \(maxValue)") } } } public func nullableValidator(_ internalValidator: @escaping (T) -> Void) -> (T?) -> Void { return { (value: T?) -> Void in if let value = value { internalValidator(value) } } } public func binaryValidator(minLength: Int?, maxLength: Int?) -> (Data) -> Void { return { (value: Data) -> Void in let length = value.count if let minLength = minLength { _assertFunc(length >= minLength, "\"\(value)\" must be at least \(minLength) bytes") } if let maxLength = maxLength { _assertFunc(length <= maxLength, "\"\(value)\" must be at most \(maxLength) bytes") } } } stone-3.3.1/stone/backends/swift_types.py000066400000000000000000000511351417406541500205160ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json import os import shutil import six from contextlib import contextmanager from stone.ir import ( is_list_type, is_numeric_type, is_string_type, is_struct_type, is_union_type, is_void_type, unwrap_nullable, ) from stone.backends.swift_helpers import ( check_route_name_conflict, fmt_class, fmt_default_value, fmt_func, fmt_var, fmt_type, ) from stone.backends.swift import ( base, fmt_serial_obj, SwiftBaseBackend, undocumented, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any _cmdline_parser = argparse.ArgumentParser(prog='swift-types-backend') _cmdline_parser.add_argument( '-r', '--route-method', help=('A string used to construct the location of a Swift method for a ' 'given route; use {ns} as a placeholder for namespace name and ' '{route} for the route name.'), ) class SwiftTypesBackend(SwiftBaseBackend): """ Generates Swift modules to represent the input Stone spec. Examples for a hypothetical 'copy' enpoint: Endpoint argument (struct): ``` open class CopyArg: CustomStringConvertible { open let fromPath: String open let toPath: String public init(fromPath: String, toPath: String) { stringValidator(pattern: "/(.|[\\r\\n])*")(value: fromPath) self.fromPath = fromPath stringValidator(pattern: "/(.|[\\r\\n])*")(value: toPath) self.toPath = toPath } open var description: String { return "\\(SerializeUtil.prepareJSONForSerialization( CopyArgSerializer().serialize(self)))" } } ``` Endpoint error (union): ``` open enum CopyError: CustomStringConvertible { case TooManyFiles case Other open var description: String { return "\\(SerializeUtil.prepareJSONForSerialization( CopyErrorSerializer().serialize(self)))" } } ``` Argument serializer (error serializer not listed): ``` open class CopyArgSerializer: JSONSerializer { public init() { } open func serialize(value: CopyArg) -> JSON { let output = [ "from_path": Serialization.serialize(value.fromPath), "to_path": Serialization.serialize(value.toPath), ] return .Dictionary(output) } open func deserialize(json: JSON) -> CopyArg { switch json { case .Dictionary(let dict): let fromPath = Serialization.deserialize(dict["from_path"] ?? .Null) let toPath = Serialization.deserialize(dict["to_path"] ?? .Null) return CopyArg(fromPath: fromPath, toPath: toPath) default: fatalError("Type error deserializing") } } } ``` """ cmdline_parser = _cmdline_parser def generate(self, api): rsrc_folder = os.path.join(os.path.dirname(__file__), 'swift_rsrc') self.logger.info('Copying StoneValidators.swift to output folder') shutil.copy(os.path.join(rsrc_folder, 'StoneValidators.swift'), self.target_folder_path) self.logger.info('Copying StoneSerializers.swift to output folder') shutil.copy(os.path.join(rsrc_folder, 'StoneSerializers.swift'), self.target_folder_path) self.logger.info('Copying StoneBase.swift to output folder') shutil.copy(os.path.join(rsrc_folder, 'StoneBase.swift'), self.target_folder_path) jazzy_cfg_path = os.path.join('../Format', 'jazzy.json') with open(jazzy_cfg_path, encoding='utf-8') as jazzy_file: jazzy_cfg = json.load(jazzy_file) for namespace in api.namespaces.values(): ns_class = fmt_class(namespace.name) with self.output_to_relative_path('{}.swift'.format(ns_class)): self._generate_base_namespace_module(api, namespace) jazzy_cfg['custom_categories'][1]['children'].append(ns_class) if namespace.routes: jazzy_cfg['custom_categories'][0]['children'].append(ns_class + 'Routes') with self.output_to_relative_path('../../../../.jazzy.json'): self.emit_raw(json.dumps(jazzy_cfg, indent=2) + '\n') def _generate_base_namespace_module(self, api, namespace): self.emit_raw(base) routes_base = 'Datatypes and serializers for the {} namespace'.format(namespace.name) self.emit_wrapped_text(routes_base, prefix='/// ', width=120) with self.block('open class {}'.format(fmt_class(namespace.name))): for data_type in namespace.linearize_data_types(): if is_struct_type(data_type): self._generate_struct_class(namespace, data_type) self.emit() elif is_union_type(data_type): self._generate_union_type(namespace, data_type) self.emit() if namespace.routes: self._generate_route_objects(api.route_schema, namespace) def _generate_struct_class(self, namespace, data_type): if data_type.doc: doc = self.process_doc(data_type.doc, self._docf) else: doc = 'The {} struct'.format(fmt_class(data_type.name)) self.emit_wrapped_text(doc, prefix='/// ', width=120) protocols = [] if not data_type.parent_type: protocols.append('CustomStringConvertible') with self.class_block(data_type, protocols=protocols): for field in data_type.fields: fdoc = self.process_doc(field.doc, self._docf) if field.doc else undocumented self.emit_wrapped_text(fdoc, prefix='/// ', width=120) self.emit('public let {}: {}'.format( fmt_var(field.name), fmt_type(field.data_type), )) self._generate_struct_init(namespace, data_type) decl = 'open var' if not data_type.parent_type else 'open override var' with self.block('{} description: String'.format(decl)): cls = fmt_class(data_type.name) + 'Serializer' self.emit('return "\\(SerializeUtil.prepareJSONForSerialization' + '({}().serialize(self)))"'.format(cls)) self._generate_struct_class_serializer(namespace, data_type) def _generate_struct_init(self, namespace, data_type): # pylint: disable=unused-argument # init method args = self._struct_init_args(data_type) if data_type.parent_type and not data_type.fields: return with self.function_block('public init', self._func_args(args)): for field in data_type.fields: v = fmt_var(field.name) validator = self._determine_validator_type(field.data_type, v) if validator: self.emit('{}({})'.format(validator, v)) self.emit('self.{0} = {0}'.format(v)) if data_type.parent_type: func_args = [(fmt_var(f.name), fmt_var(f.name)) for f in data_type.parent_type.all_fields] self.emit('super.init({})'.format(self._func_args(func_args))) def _determine_validator_type(self, data_type, value): data_type, nullable = unwrap_nullable(data_type) if is_list_type(data_type): item_validator = self._determine_validator_type(data_type.data_type, value) if item_validator: v = "arrayValidator({})".format( self._func_args([ ("minItems", data_type.min_items), ("maxItems", data_type.max_items), ("itemValidator", item_validator), ]) ) else: return None elif is_numeric_type(data_type): v = "comparableValidator({})".format( self._func_args([ ("minValue", data_type.min_value), ("maxValue", data_type.max_value), ]) ) elif is_string_type(data_type): pat = data_type.pattern if data_type.pattern else None pat = pat.encode('unicode_escape').replace(six.ensure_binary("\""), six.ensure_binary("\\\"")) if pat else pat v = "stringValidator({})".format( self._func_args([ ("minLength", data_type.min_length), ("maxLength", data_type.max_length), ("pattern", '"{}"'.format(six.ensure_str(pat)) if pat else None), ]) ) else: return None if nullable: v = "nullableValidator({})".format(v) return v def _generate_enumerated_subtype_serializer(self, namespace, # pylint: disable=unused-argument data_type): with self.block('switch value'): for tags, subtype in data_type.get_all_subtypes_with_tags(): assert len(tags) == 1, tags tag = tags[0] tagvar = fmt_var(tag) self.emit('case let {} as {}:'.format( tagvar, fmt_type(subtype) )) with self.indent(): block_txt = 'for (k, v) in Serialization.getFields({}.serialize({}))'.format( fmt_serial_obj(subtype), tagvar, ) with self.block(block_txt): self.emit('output[k] = v') self.emit('output[".tag"] = .str("{}")'.format(tag)) self.emit('default: fatalError("Tried to serialize unexpected subtype")') def _generate_struct_base_class_deserializer(self, namespace, data_type): args = [] for field in data_type.all_fields: var = fmt_var(field.name) value = 'dict["{}"]'.format(field.name) self.emit('let {} = {}.deserialize({} ?? {})'.format( var, fmt_serial_obj(field.data_type), value, fmt_default_value(namespace, field) if field.has_default else '.null' )) args.append((var, var)) self.emit('return {}({})'.format( fmt_class(data_type.name), self._func_args(args) )) def _generate_enumerated_subtype_deserializer(self, namespace, data_type): self.emit('let tag = Serialization.getTag(dict)') with self.block('switch tag'): for tags, subtype in data_type.get_all_subtypes_with_tags(): assert len(tags) == 1, tags tag = tags[0] self.emit('case "{}":'.format(tag)) with self.indent(): self.emit('return {}.deserialize(json)'.format(fmt_serial_obj(subtype))) self.emit('default:') with self.indent(): if data_type.is_catch_all(): self._generate_struct_base_class_deserializer(namespace, data_type) else: self.emit('fatalError("Unknown tag \\(tag)")') def _generate_struct_class_serializer(self, namespace, data_type): with self.serializer_block(data_type): with self.serializer_func(data_type): if not data_type.all_fields: self.emit('let output = [String: JSON]()') else: intro = 'var' if data_type.has_enumerated_subtypes() else 'let' self.emit("{} output = [ ".format(intro)) for field in data_type.all_fields: self.emit('"{}": {}.serialize(value.{}),'.format( field.name, fmt_serial_obj(field.data_type), fmt_var(field.name) )) self.emit(']') if data_type.has_enumerated_subtypes(): self._generate_enumerated_subtype_serializer(namespace, data_type) self.emit('return .dictionary(output)') with self.deserializer_func(data_type): with self.block("switch json"): dict_name = "let dict" if data_type.all_fields else "_" self.emit("case .dictionary({}):".format(dict_name)) with self.indent(): if data_type.has_enumerated_subtypes(): self._generate_enumerated_subtype_deserializer(namespace, data_type) else: self._generate_struct_base_class_deserializer(namespace, data_type) self.emit("default:") with self.indent(): self.emit('fatalError("Type error deserializing")') def _format_tag_type(self, namespace, data_type): # pylint: disable=unused-argument if is_void_type(data_type): return '' else: return '({})'.format(fmt_type(data_type)) def _generate_union_type(self, namespace, data_type): if data_type.doc: doc = self.process_doc(data_type.doc, self._docf) else: doc = 'The {} union'.format(fmt_class(data_type.name)) self.emit_wrapped_text(doc, prefix='/// ', width=120) class_type = fmt_class(data_type.name) with self.block('public enum {}: CustomStringConvertible'.format(class_type)): for field in data_type.all_fields: typ = self._format_tag_type(namespace, field.data_type) fdoc = self.process_doc(field.doc, self._docf) if field.doc else 'An unspecified error.' self.emit_wrapped_text(fdoc, prefix='/// ', width=120) self.emit('case {}{}'.format(fmt_var(field.name), typ)) self.emit() with self.block('public var description: String'): cls = class_type + 'Serializer' self.emit('return "\\(SerializeUtil.prepareJSONForSerialization' + '({}().serialize(self)))"'.format(cls)) self._generate_union_serializer(data_type) def _tag_type(self, data_type, field): return "{}.{}".format( fmt_class(data_type.name), fmt_var(field.name) ) def _generate_union_serializer(self, data_type): with self.serializer_block(data_type): with self.serializer_func(data_type), self.block('switch value'): for field in data_type.all_fields: field_type = field.data_type case = '.{}{}'.format(fmt_var(field.name), '' if is_void_type(field_type) else '(let arg)') self.emit('case {}:'.format(case)) with self.indent(): if is_void_type(field_type): self.emit('var d = [String: JSON]()') elif (is_struct_type(field_type) and not field_type.has_enumerated_subtypes()): self.emit('var d = Serialization.getFields({}.serialize(arg))'.format( fmt_serial_obj(field_type))) else: self.emit('var d = ["{}": {}.serialize(arg)]'.format( field.name, fmt_serial_obj(field_type))) self.emit('d[".tag"] = .str("{}")'.format(field.name)) self.emit('return .dictionary(d)') with self.deserializer_func(data_type): with self.block("switch json"): self.emit("case .dictionary(let d):") with self.indent(): self.emit('let tag = Serialization.getTag(d)') with self.block('switch tag'): for field in data_type.all_fields: field_type = field.data_type self.emit('case "{}":'.format(field.name)) tag_type = self._tag_type(data_type, field) with self.indent(): if is_void_type(field_type): self.emit('return {}'.format(tag_type)) else: if (is_struct_type(field_type) and not field_type.has_enumerated_subtypes()): subdict = 'json' else: subdict = 'd["{}"] ?? .null'.format(field.name) self.emit('let v = {}.deserialize({})'.format( fmt_serial_obj(field_type), subdict )) self.emit('return {}(v)'.format(tag_type)) self.emit('default:') with self.indent(): if data_type.catch_all_field: self.emit('return {}'.format( self._tag_type(data_type, data_type.catch_all_field) )) else: self.emit('fatalError("Unknown tag \\(tag)")') self.emit("default:") with self.indent(): self.emit('fatalError("Failed to deserialize")') @contextmanager def serializer_block(self, data_type): with self.class_block(fmt_class(data_type.name) + 'Serializer', protocols=['JSONSerializer']): self.emit("public init() { }") yield @contextmanager def serializer_func(self, data_type): with self.function_block('open func serialize', args=self._func_args([('_ value', fmt_class(data_type.name))]), return_type='JSON'): yield @contextmanager def deserializer_func(self, data_type): with self.function_block('open func deserialize', args=self._func_args([('_ json', 'JSON')]), return_type=fmt_class(data_type.name)): yield def _generate_route_objects(self, route_schema, namespace): check_route_name_conflict(namespace) self.emit() self.emit('/// Stone Route Objects') self.emit() for route in namespace.routes: var_name = fmt_func(route.name, route.version) with self.block('static let {} = Route('.format(var_name), delim=(None, None), after=')'): self.emit('name: \"{}\",'.format(route.name)) self.emit('version: {},'.format(route.version)) self.emit('namespace: \"{}\",'.format(namespace.name)) self.emit('deprecated: {},'.format('true' if route.deprecated is not None else 'false')) self.emit('argSerializer: {},'.format(fmt_serial_obj(route.arg_data_type))) self.emit('responseSerializer: {},'.format(fmt_serial_obj(route.result_data_type))) self.emit('errorSerializer: {},'.format(fmt_serial_obj(route.error_data_type))) attrs = [] for field in route_schema.fields: attr_key = field.name attr_val = ("\"{}\"".format(route.attrs.get(attr_key)) if route.attrs.get(attr_key) else 'nil') attrs.append('\"{}\": {}'.format(attr_key, attr_val)) self.generate_multiline_list( attrs, delim=('attrs: [', ']'), compact=True) stone-3.3.1/stone/backends/tsd_client.py000066400000000000000000000172231417406541500202660ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import os import re _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any from stone.backend import CodeBackend from stone.backends.tsd_helpers import ( check_route_name_conflict, fmt_error_type, fmt_func, fmt_tag, fmt_type, get_data_types_for_namespace, ) from stone.ir import Void _cmdline_parser = argparse.ArgumentParser(prog='tsd-client-backend') _cmdline_parser.add_argument( 'template', help=('A template to use when generating the TypeScript definition file.') ) _cmdline_parser.add_argument( 'filename', help=('The name to give the single TypeScript definition file to contain ' 'all of the emitted types.'), ) _cmdline_parser.add_argument( '-t', '--template-string', type=str, default='ROUTES', help=('The name of the template string to replace with route definitions. ' 'Defaults to ROUTES, which replaces the string /*ROUTES*/ with route ' 'definitions.') ) _cmdline_parser.add_argument( '-i', '--indent-level', type=int, default=1, help=('Indentation level to emit types at. Routes are automatically ' 'indented one level further than this.') ) _cmdline_parser.add_argument( '-s', '--spaces-per-indent', type=int, default=2, help=('Number of spaces to use per indentation level.') ) _cmdline_parser.add_argument( '--wrap-response-in', type=str, default='', help=('Wraps the response in a response class') ) _cmdline_parser.add_argument( '--wrap-error-in', type=str, default='', help=('Wraps the error in an error class') ) _cmdline_parser.add_argument( '--import-namespaces', default=False, action='store_true', help=('Adds an import statement at the top of the file to import each ' 'namespace from the as a named import. Must be used in conjunction ' 'with the --export-namespaces command when generating the ts_types.') ) _cmdline_parser.add_argument( '--import-template-string', type=str, default='IMPORT', help=('The name of the template string to replace with import statement. ' 'Defaults to IMPORT, which replaces the string /*IMPORT*/ with import.') ) _cmdline_parser.add_argument( '--types-file', type=str, default='', help=('If using the --import-namespaces flag, this is the file that contains ' 'the named exports to import here.') ) _header = """\ // Auto-generated by Stone, do not modify. """ class TSDClientBackend(CodeBackend): """Generates a TypeScript definition file with routes defined.""" cmdline_parser = _cmdline_parser preserve_aliases = True def generate(self, api): spaces_per_indent = self.args.spaces_per_indent indent_level = self.args.indent_level template_path = os.path.join(self.target_folder_path, self.args.template) template_string = self.args.template_string with self.output_to_relative_path(self.args.filename): if os.path.isfile(template_path): with open(template_path, 'r', encoding='utf-8') as template_file: template = template_file.read() else: raise AssertionError('TypeScript template file does not exist.') # /*ROUTES*/ r_match = re.search("/\\*%s\\*/" % (template_string), template) if not r_match: raise AssertionError( 'Missing /*%s*/ in TypeScript template file.' % template_string) r_start = r_match.start() r_end = r_match.end() r_ends_with_newline = template[r_end - 1] == '\n' t_end = len(template) t_ends_with_newline = template[t_end - 1] == '\n' if self.args.import_namespaces: import_template_string = self.args.import_template_string import_from_file = self.args.types_file # /*IMPORT*/ i_match = re.search("/\\*%s\\*/" % (import_template_string), template) if not i_match: raise AssertionError( 'Missing /*%s*/ in TypeScript template file.' % import_template_string) i_start = i_match.start() i_end = i_match.end() i_ends_with_newline = template[i_end - 1] == '\n' t_end = len(template) t_ends_with_newline = template[t_end - 1] == '\n' self.emit_raw(template[0:i_start] + ('\n' if not i_ends_with_newline else '')) self._generate_import(api, import_from_file) self.emit_raw(template[i_end + 1:r_end] + ('\n' if not r_ends_with_newline else '')) else: self.emit_raw(template[0:r_start] + ('\n' if not r_ends_with_newline else '')) self._generate_routes(api, spaces_per_indent, indent_level) self.emit_raw(template[r_end + 1:t_end] + ('\n' if not t_ends_with_newline else '')) def _generate_import(self, api, type_file): # identify which routes belong to namespaces_with_types = filter( lambda namespace: len(get_data_types_for_namespace(namespace)) != 0, api.namespaces.values()) namespaces = ", ".join(map(lambda namespace: namespace.name, namespaces_with_types)) self.emit("import { %s } from '%s';" % (namespaces, type_file)) def _generate_routes(self, api, spaces_per_indent, indent_level): with self.indent(dent=spaces_per_indent * (indent_level + 1)): for namespace in api.namespaces.values(): # first check for route name conflict check_route_name_conflict(namespace) for route in namespace.routes: self._generate_route( namespace, route) def _generate_route(self, namespace, route): function_name = fmt_func(namespace.name + '_' + route.name, route.version) self.emit() self.emit('/**') if route.doc: self.emit_wrapped_text(self.process_doc(route.doc, self._docf), prefix=' * ') self.emit(' *') self.emit_wrapped_text('When an error occurs, the route rejects the promise with type %s.' % fmt_error_type(route.error_data_type, wrap_error_in=self.args.wrap_error_in), prefix=' * ') if route.deprecated: self.emit(' * @deprecated') if route.arg_data_type.__class__ != Void: self.emit(' * @param arg The request parameters.') self.emit(' */') return_type = None if self.args.wrap_response_in: return_type = 'Promise<%s<%s>>;' % (self.args.wrap_response_in, fmt_type(route.result_data_type)) else: return_type = 'Promise<%s>;' % (fmt_type(route.result_data_type)) arg = '' if route.arg_data_type.__class__ != Void: arg = 'arg: %s' % fmt_type(route.arg_data_type) self.emit('public %s(%s): %s' % (function_name, arg, return_type)) def _docf(self, tag, val): """ Callback to process documentation references. """ return fmt_tag(None, tag, val) stone-3.3.1/stone/backends/tsd_helpers.py000066400000000000000000000141521417406541500204500ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from stone.backend import Backend from stone.ir.api import ApiNamespace from stone.ir import ( Boolean, Bytes, Float32, Float64, Int32, Int64, List, String, Timestamp, UInt32, UInt64, Void, is_alias, is_list_type, is_struct_type, is_map_type, is_user_defined_type, ) from stone.backends.helpers import ( fmt_camel, ) _base_type_table = { Boolean: 'boolean', Bytes: 'string', Float32: 'number', Float64: 'number', Int32: 'number', Int64: 'number', List: 'Array', String: 'string', UInt32: 'number', UInt64: 'number', Timestamp: 'Timestamp', Void: 'void', } def fmt_error_type(data_type, inside_namespace=None, wrap_error_in=''): """ Converts the error type into a TypeScript type. inside_namespace should be set to the namespace that the reference occurs in, or None if this parameter is not relevant. """ return '%s<%s>' % ( (wrap_error_in if (wrap_error_in != '') else 'Error'), fmt_type(data_type, inside_namespace) ) def fmt_type_name(data_type, inside_namespace=None): """ Produces a TypeScript type name for the given data type. inside_namespace should be set to the namespace that the reference occurs in, or None if this parameter is not relevant. """ if is_user_defined_type(data_type) or is_alias(data_type): if data_type.namespace == inside_namespace: return data_type.name else: return '%s.%s' % (data_type.namespace.name, data_type.name) else: fmted_type = _base_type_table.get(data_type.__class__, 'Object') if is_list_type(data_type): fmted_type += '<' + fmt_type(data_type.data_type, inside_namespace) + '>' elif is_map_type(data_type): key_data_type = _base_type_table.get(data_type.key_data_type, 'string') value_data_type = fmt_type_name(data_type.value_data_type, inside_namespace) fmted_type = '{[key: %s]: %s}' % (key_data_type, value_data_type) return fmted_type def fmt_polymorphic_type_reference(data_type, inside_namespace=None): """ Produces a TypeScript type name for the meta-type that refers to the given struct, which belongs to an enumerated subtypes tree. This meta-type contains the .tag field that lets developers discriminate between subtypes. """ # NOTE: These types are not properly namespaced, so there could be a conflict # with other user-defined types. If this ever surfaces as a problem, we # can defer emitting these types until the end, and emit them in a # nested namespace (e.g., files.references.MetadataReference). return fmt_type_name(data_type, inside_namespace) + "Reference" def fmt_type(data_type, inside_namespace=None): """ Returns a TypeScript type annotation for a data type. May contain a union of enumerated subtypes. inside_namespace should be set to the namespace that the type reference occurs in, or None if this parameter is not relevant. """ if is_struct_type(data_type) and data_type.has_enumerated_subtypes(): possible_types = [] possible_subtypes = data_type.get_all_subtypes_with_tags() for _, subtype in possible_subtypes: possible_types.append(fmt_polymorphic_type_reference(subtype, inside_namespace)) if data_type.is_catch_all(): possible_types.append(fmt_polymorphic_type_reference(data_type, inside_namespace)) return fmt_union(possible_types) else: return fmt_type_name(data_type, inside_namespace) def fmt_union(type_strings): """ Returns a union type of the given types. """ return '|'.join(type_strings) if len(type_strings) > 1 else type_strings[0] def fmt_func(name, version): if version == 1: return fmt_camel(name) return fmt_camel(name) + 'V{}'.format(version) def fmt_var(name): return fmt_camel(name) def fmt_tag(cur_namespace, tag, val): """ Processes a documentation reference. """ if tag == 'type': fq_val = val if '.' not in val and cur_namespace is not None: fq_val = cur_namespace.name + '.' + fq_val return fq_val elif tag == 'route': if ':' in val: val, version = val.split(':', 1) version = int(version) else: version = 1 return fmt_func(val, version) + "()" elif tag == 'link': anchor, link = val.rsplit(' ', 1) # There's no way to have links in TSDoc, so simply use JSDoc's formatting. # It's entirely possible some editors support this. return '[%s]{@link %s}' % (anchor, link) elif tag == 'val': # Value types seem to match JavaScript (true, false, null) return val elif tag == 'field': return val else: raise RuntimeError('Unknown doc ref tag %r' % tag) def check_route_name_conflict(namespace): """ Check name conflicts among generated route definitions. Raise a runtime exception when a conflict is encountered. """ route_by_name = {} for route in namespace.routes: route_name = fmt_func(route.name, route.version) if route_name in route_by_name: other_route = route_by_name[route_name] raise RuntimeError( 'There is a name conflict between {!r} and {!r}'.format(other_route, route)) route_by_name[route_name] = route def generate_imports_for_referenced_namespaces(backend, namespace, module_name_prefix): # type: (Backend, ApiNamespace, str) -> None imported_namespaces = namespace.get_imported_namespaces() if not imported_namespaces: return for ns in imported_namespaces: backend.emit( "import * as {namespace_name} from '{module_name_prefix}{namespace_name}';".format( module_name_prefix=module_name_prefix, namespace_name=ns.name ) ) backend.emit() def get_data_types_for_namespace(namespace): return namespace.data_types + namespace.aliases stone-3.3.1/stone/backends/tsd_types.py000066400000000000000000000512401417406541500201510ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import json import os import re import six import sys _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any from stone.ir import ApiNamespace from stone.ir import ( is_alias, is_struct_type, is_union_type, is_user_defined_type, is_void_type, unwrap_nullable, ) from stone.backend import CodeBackend from stone.backends.helpers import ( fmt_pascal, ) from stone.backends.tsd_helpers import ( fmt_polymorphic_type_reference, fmt_tag, fmt_type, fmt_type_name, fmt_union, generate_imports_for_referenced_namespaces, get_data_types_for_namespace, ) _cmdline_parser = argparse.ArgumentParser(prog='tsd-types-backend') _cmdline_parser.add_argument( 'template', help=('A template to use when generating the TypeScript definition file. ' 'Replaces the string /*TYPES*/ with stone type definitions.') ) _cmdline_parser.add_argument( 'filename', nargs='?', help=('The name of the generated typeScript definition file that contains ' 'all of the emitted types.'), ) _cmdline_parser.add_argument( '--exclude_error_types', default=False, action='store_true', help='If true, the output will exclude the interface for Error type.', ) _cmdline_parser.add_argument( '-e', '--extra-arg', action='append', type=str, default=[], help=("Additional argument to add to a route's argument based " "on if the route has a certain attribute set. Format (JSON): " '{"match": ["ROUTE_ATTR", ROUTE_VALUE_TO_MATCH], ' '"arg_name": "ARG_NAME", "arg_type": "ARG_TYPE", ' '"arg_docstring": "ARG_DOCSTRING"}'), ) _cmdline_parser.add_argument( '-i', '--indent-level', type=int, default=1, help=('Indentation level to emit types at. Routes are automatically ' 'indented one level further than this.') ) _cmdline_parser.add_argument( '-s', '--spaces-per-indent', type=int, default=2, help=('Number of spaces to use per indentation level.') ) _cmdline_parser.add_argument( '-p', '--module-name-prefix', type=str, default='', help=('Prefix for data type module names. ' 'This is useful for repo which requires absolute path as ' 'module name') ) _cmdline_parser.add_argument( '--export-namespaces', default=False, action='store_true', help=('Adds the export tag to each namespace.' 'This is useful is you are not placing each namespace ' 'inside of a module and want to export each namespace individually') ) _header = """\ // Auto-generated by Stone, do not modify. """ _types_header = """\ /** * An Error object returned from a route. */ interface Error { \t// Text summary of the error. \terror_summary: string; \t// The error object. \terror: T; \t// User-friendly error message. \tuser_message: UserMessage; } /** * User-friendly error message. */ interface UserMessage { \t// The message. \ttext: string; \t// The locale of the message. \tlocale: string; } """ _timestamp_definition = "type Timestamp = string;" class TSDTypesBackend(CodeBackend): """ Generates a single TypeScript definition file with all of the types defined, organized as namespaces, if a filename is provided in input arguments. Otherwise generates one declaration file for each namespace with the corresponding typescript definitions. If a single output file is generated, a top level type definition will be added for the Timestamp data type. Otherwise, each namespace will have the type definition for Timestamp. Also, note that namespace definitions are emitted as declaration files. Hence any template provided as argument must not have a top level declare statement. If namespaces are emitted into a single file, the template file can be used to wrap them around a declare statement. """ cmdline_parser = _cmdline_parser preserve_aliases = True # Instance var of the current namespace being generated cur_namespace = None # type: typing.Optional[ApiNamespace] # Instance var to denote if one file is output for each namespace. split_by_namespace = False def generate(self, api): extra_args = self._parse_extra_args(api, self.args.extra_arg) template = self._read_template() if self.args.filename: self._generate_base_namespace_module(api.namespaces.values(), self.args.filename, template, extra_args, exclude_error_types=self.args.exclude_error_types) else: self.split_by_namespace = True for namespace in api.namespaces.values(): filename = '{}.d.ts'.format(namespace.name) self._generate_base_namespace_module( [namespace], filename, template, extra_args, exclude_error_types=self.args.exclude_error_types) def _read_template(self): template_path = os.path.join(self.target_folder_path, self.args.template) if os.path.isfile(template_path): with open(template_path, 'r', encoding='utf-8') as template_file: return template_file.read() else: raise AssertionError('TypeScript template file does not exist.') def _generate_base_namespace_module(self, namespace_list, filename, template, extra_args, exclude_error_types=False): # Skip namespaces that do not contain types. if all([len(get_data_types_for_namespace(ns)) == 0 for ns in namespace_list]): return spaces_per_indent = self.args.spaces_per_indent indent_level = self.args.indent_level with self.output_to_relative_path(filename): # /*TYPES*/ t_match = re.search("/\\*TYPES\\*/", template) if not t_match: raise AssertionError('Missing /*TYPES*/ in TypeScript template file.') t_start = t_match.start() t_end = t_match.end() t_ends_with_newline = template[t_end - 1] == '\n' temp_end = len(template) temp_ends_with_newline = template[temp_end - 1] == '\n' self.emit_raw(template[0:t_start] + ("\n" if not t_ends_with_newline else '')) indent = spaces_per_indent * indent_level indent_spaces = (' ' * indent) with self.indent(dent=indent): if not exclude_error_types: indented_types_header = indent_spaces + ( ('\n' + indent_spaces) .join(_types_header.split('\n')) .replace('\t', ' ' * spaces_per_indent) ) self.emit_raw(indented_types_header + '\n') if not self.split_by_namespace: self.emit(_timestamp_definition) self.emit() for namespace in namespace_list: self._generate_types(namespace, spaces_per_indent, extra_args) self.emit_raw(template[t_end + 1:temp_end] + ("\n" if not temp_ends_with_newline else '')) def _generate_types(self, namespace, spaces_per_indent, extra_args): self.cur_namespace = namespace # Count aliases as data types too! data_types = get_data_types_for_namespace(namespace) # Skip namespaces that do not contain types. if len(data_types) == 0: return if self.split_by_namespace: generate_imports_for_referenced_namespaces( backend=self, namespace=namespace, module_name_prefix=self.args.module_name_prefix ) if namespace.doc: self._emit_tsdoc_header(namespace.doc) self.emit_wrapped_text(self._get_top_level_declaration(namespace.name)) with self.indent(dent=spaces_per_indent): for data_type in data_types: self._generate_type(data_type, spaces_per_indent, extra_args.get(data_type, [])) if self.split_by_namespace: with self.indent(dent=spaces_per_indent): # TODO(Pranay): May avoid adding an unused definition if needed. self.emit(_timestamp_definition) self.emit('}') self.emit() def _get_top_level_declaration(self, name): if self.split_by_namespace: # Use module for when emitting declaration files. return "declare module '%s%s' {" % (self.args.module_name_prefix, name) else: if self.args.export_namespaces: return "export namespace %s {" % name else: # Use namespace for organizing code with-in the file. return "namespace %s {" % name def _parse_extra_args(self, api, extra_args_raw): """ Parses extra arguments into a map keyed on particular data types. """ extra_args = {} def invalid(msg, extra_arg_raw): print('Invalid --extra-arg:%s: %s' % (msg, extra_arg_raw), file=sys.stderr) sys.exit(1) for extra_arg_raw in extra_args_raw: try: extra_arg = json.loads(extra_arg_raw) except ValueError as e: invalid(str(e), extra_arg_raw) # Validate extra_arg JSON blob if 'match' not in extra_arg: invalid('No match key', extra_arg_raw) elif (not isinstance(extra_arg['match'], list) or len(extra_arg['match']) != 2): invalid('match key is not a list of two strings', extra_arg_raw) elif (not isinstance(extra_arg['match'][0], six.text_type) or not isinstance(extra_arg['match'][1], six.text_type)): print(type(extra_arg['match'][0])) invalid('match values are not strings', extra_arg_raw) elif 'arg_name' not in extra_arg: invalid('No arg_name key', extra_arg_raw) elif not isinstance(extra_arg['arg_name'], six.text_type): invalid('arg_name is not a string', extra_arg_raw) elif 'arg_type' not in extra_arg: invalid('No arg_type key', extra_arg_raw) elif not isinstance(extra_arg['arg_type'], six.text_type): invalid('arg_type is not a string', extra_arg_raw) elif ('arg_docstring' in extra_arg and not isinstance(extra_arg['arg_docstring'], six.text_type)): invalid('arg_docstring is not a string', extra_arg_raw) attr_key, attr_val = extra_arg['match'][0], extra_arg['match'][1] extra_args.setdefault(attr_key, {})[attr_val] = \ (extra_arg['arg_name'], extra_arg['arg_type'], extra_arg.get('arg_docstring')) # Extra arguments, keyed on data type objects. extra_args_for_types = {} # Locate data types that contain extra arguments for namespace in api.namespaces.values(): for route in namespace.routes: extra_parameters = [] if is_user_defined_type(route.arg_data_type): for attr_key in route.attrs: if attr_key not in extra_args: continue attr_val = route.attrs[attr_key] if attr_val in extra_args[attr_key]: extra_parameters.append(extra_args[attr_key][attr_val]) if len(extra_parameters) > 0: extra_args_for_types[route.arg_data_type] = extra_parameters return extra_args_for_types def _emit_tsdoc_header(self, docstring): self.emit('/**') self.emit_wrapped_text(self.process_doc(docstring, self._docf), prefix=' * ') self.emit(' */') def _generate_type(self, data_type, indent_spaces, extra_args): """ Generates a TypeScript type for the given type. """ if is_alias(data_type): self._generate_alias_type(data_type) elif is_struct_type(data_type): self._generate_struct_type(data_type, indent_spaces, extra_args) elif is_union_type(data_type): self._generate_union_type(data_type, indent_spaces) def _generate_alias_type(self, alias_type): """ Generates a TypeScript type for a stone alias. """ namespace = alias_type.namespace self.emit('export type %s = %s;' % (fmt_type_name(alias_type, namespace), fmt_type_name(alias_type.data_type, namespace))) self.emit() def _generate_struct_type(self, struct_type, indent_spaces, extra_parameters): """ Generates a TypeScript interface for a stone struct. """ namespace = struct_type.namespace if struct_type.doc: self._emit_tsdoc_header(struct_type.doc) parent_type = struct_type.parent_type extends_line = ' extends %s' % fmt_type_name(parent_type, namespace) if parent_type else '' self.emit('export interface %s%s {' % (fmt_type_name(struct_type, namespace), extends_line)) with self.indent(dent=indent_spaces): for param_name, param_type, param_docstring in extra_parameters: if param_docstring: self._emit_tsdoc_header(param_docstring) # Making all extra args optional parameters self.emit('%s?: %s;' % (param_name, param_type)) for field in struct_type.fields: doc = field.doc field_type, nullable = unwrap_nullable(field.data_type) field_ts_type = fmt_type(field_type, namespace) optional = nullable or field.has_default if field.has_default: # doc may be None. If it is not empty, add newlines # before appending to it. doc = doc + '\n\n' if doc else '' doc = "Defaults to %s." % field.default if doc: self._emit_tsdoc_header(doc) # Translate nullable types into optional properties. field_name = '%s?' % field.name if optional else field.name self.emit('%s: %s;' % (field_name, field_ts_type)) self.emit('}') self.emit() # Some structs can explicitly list their subtypes. These structs have a .tag field that # indicate which subtype they are, which is only present when a type reference is # ambiguous. # Emit a special interface that contains this extra field, and refer to it whenever we # encounter a reference to a type with enumerated subtypes. if struct_type.is_member_of_enumerated_subtypes_tree(): if struct_type.has_enumerated_subtypes(): # This struct is the parent to multiple subtypes. Determine all of the possible # values of the .tag property. tag_values = [] for tags, _ in struct_type.get_all_subtypes_with_tags(): for tag in tags: tag_values.append('"%s"' % tag) tag_union = fmt_union(tag_values) self._emit_tsdoc_header('Reference to the %s polymorphic type. Contains a .tag ' 'property to let you discriminate between possible ' 'subtypes.' % fmt_type_name(struct_type, namespace)) self.emit('export interface %s extends %s {' % (fmt_polymorphic_type_reference(struct_type, namespace), fmt_type_name(struct_type, namespace))) with self.indent(dent=indent_spaces): self._emit_tsdoc_header('Tag identifying the subtype variant.') self.emit('\'.tag\': %s;' % tag_union) self.emit('}') self.emit() else: # This struct is a particular subtype. Find the applicable .tag value from the # parent type, which may be an arbitrary number of steps up the inheritance # hierarchy. parent = struct_type.parent_type while not parent.has_enumerated_subtypes(): parent = parent.parent_type # parent now contains the closest parent type in the inheritance hierarchy that has # enumerated subtypes. Determine which subtype this is. for subtype in parent.get_enumerated_subtypes(): if subtype.data_type == struct_type: self._emit_tsdoc_header('Reference to the %s type, identified by the ' 'value of the .tag property.' % fmt_type_name(struct_type, namespace)) self.emit('export interface %s extends %s {' % (fmt_polymorphic_type_reference(struct_type, namespace), fmt_type_name(struct_type, namespace))) with self.indent(dent=indent_spaces): self._emit_tsdoc_header('Tag identifying this subtype variant. This ' 'field is only present when needed to ' 'discriminate between multiple possible ' 'subtypes.') self.emit_wrapped_text('\'.tag\': \'%s\';' % subtype.name) self.emit('}') self.emit() break def _generate_union_type(self, union_type, indent_spaces): """ Generates a TypeScript interface for a stone union. """ # Emit an interface for each variant. TypeScript 2.0 supports these tagged unions. # https://github.com/Microsoft/TypeScript/wiki/What%27s-new-in-TypeScript#tagged-union-types parent_type = union_type.parent_type namespace = union_type.namespace union_type_name = fmt_type_name(union_type, namespace) variant_type_names = [] if parent_type: variant_type_names.append(fmt_type_name(parent_type, namespace)) def _is_struct_without_enumerated_subtypes(data_type): """ :param data_type: any data type. :return: True if the given data type is a struct which has no enumerated subtypes. """ return is_struct_type(data_type) and ( not data_type.has_enumerated_subtypes()) for variant in union_type.fields: if variant.doc: self._emit_tsdoc_header(variant.doc) variant_name = '%s%s' % (union_type_name, fmt_pascal(variant.name)) variant_type_names.append(variant_name) is_struct_without_enumerated_subtypes = _is_struct_without_enumerated_subtypes( variant.data_type) if is_struct_without_enumerated_subtypes: self.emit('export interface %s extends %s {' % ( variant_name, fmt_type(variant.data_type, namespace))) else: self.emit('export interface %s {' % variant_name) with self.indent(dent=indent_spaces): # Since field contains non-alphanumeric character, we need to enclose # it in quotation marks. self.emit("'.tag': '%s';" % variant.name) if is_void_type(variant.data_type) is False and ( not is_struct_without_enumerated_subtypes ): self.emit("%s: %s;" % (variant.name, fmt_type(variant.data_type, namespace))) self.emit('}') self.emit() if union_type.doc: self._emit_tsdoc_header(union_type.doc) self.emit('export type %s = %s;' % (union_type_name, ' | '.join(variant_type_names))) self.emit() def _docf(self, tag, val): """ Callback to process documentation references. """ return fmt_tag(self.cur_namespace, tag, val) stone-3.3.1/stone/cli.py000066400000000000000000000333461417406541500151370ustar00rootroot00000000000000""" A command-line interface for StoneAPI. """ from __future__ import absolute_import, division, print_function, unicode_literals import codecs import imp # pylint: disable=deprecated-module import io import json import logging import os import six import sys import traceback from .cli_helpers import parse_route_attr_filter from .compiler import ( BackendException, Compiler, ) from .frontend.exception import InvalidSpec from .frontend.frontend import specs_to_ir _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any # These backends come by default _builtin_backends = ( 'obj_c_client', 'obj_c_types', 'obj_c_tests', 'js_client', 'js_types', 'tsd_client', 'tsd_types', 'python_types', 'python_type_stubs', 'python_client', 'swift_types', 'swift_client', ) # The parser for command line arguments _cmdline_description = ( 'Write your APIs in Stone. Use backends to translate your specification ' 'into a target language or format. The following describes arguments to ' 'the Stone CLI. To specify arguments that are specific to a backend, ' 'add "--" followed by arguments. For example, "stone python_client . ' 'example.spec -- -h".' ) _cmdline_parser = argparse.ArgumentParser(description=_cmdline_description) _cmdline_parser.add_argument( '-v', '--verbose', action='count', help='Print debugging statements.', ) _backend_help = ( 'Either the name of a built-in backend or the path to a backend ' 'module. Paths to backend modules must end with a .stoneg.py extension. ' 'The following backends are built-in: ' + ', '.join(_builtin_backends)) _cmdline_parser.add_argument( 'backend', type=six.text_type, help=_backend_help, ) _cmdline_parser.add_argument( 'output', type=six.text_type, help='The folder to save generated files to.', ) _cmdline_parser.add_argument( 'spec', nargs='*', type=six.text_type, help=('Path to API specifications. Each must have a .stone extension. ' 'If omitted or set to "-", the spec is read from stdin. Multiple ' 'namespaces can be provided over stdin by concatenating multiple ' 'specs together.'), ) _cmdline_parser.add_argument( '--clean-build', action='store_true', help='The path to the template SDK for the target language.', ) _cmdline_parser.add_argument( '-f', '--filter-by-route-attr', type=six.text_type, help=('Removes routes that do not match the expression. The expression ' 'must specify a route attribute on the left-hand side and a value ' 'on the right-hand side. Use quotes for strings and bytes. The only ' 'supported operators are "=" and "!=". For example, if "hide" is a ' 'route attribute, we can use this filter: "hide!=true". You can ' 'combine multiple expressions with "and"/"or" and use parentheses ' 'to enforce precedence.'), ) _cmdline_parser.add_argument( '-r', '--route-whitelist-filter', type=six.text_type, help=('Restrict datatype generation to only the routes specified in the whitelist ' 'and their dependencies. Input should be a file containing a JSON dict with ' 'the following form: {"route_whitelist": {}, "datatype_whitelist": {}} ' 'where each object maps namespaces to lists of routes or datatypes to whitelist.'), ) _cmdline_parser.add_argument( '-a', '--attribute', action='append', type=str, default=[], help=('Route attributes that the backend will have access to and ' 'presumably expose in generated code. Use ":all" to select all ' 'attributes defined in stone_cfg.Route. Note that you can filter ' '(-f) by attributes that are not listed here.'), ) _filter_ns_group = _cmdline_parser.add_mutually_exclusive_group() _filter_ns_group.add_argument( '-w', '--whitelist-namespace-routes', action='append', type=str, default=[], help='If set, backends will only see the specified namespaces as having routes.', ) _filter_ns_group.add_argument( '-b', '--blacklist-namespace-routes', action='append', type=str, default=[], help='If set, backends will not see any routes for the specified namespaces.', ) def main(): """The entry point for the program.""" if '--' in sys.argv: cli_args = sys.argv[1:sys.argv.index('--')] backend_args = sys.argv[sys.argv.index('--') + 1:] else: cli_args = sys.argv[1:] backend_args = [] args = _cmdline_parser.parse_args(cli_args) debug = False if args.verbose is None: logging_level = logging.WARNING elif args.verbose == 1: logging_level = logging.INFO elif args.verbose == 2: logging_level = logging.DEBUG debug = True else: print('error: I can only be so garrulous, try -vv.', file=sys.stderr) sys.exit(1) logging.basicConfig(level=logging_level) if args.spec and args.spec[0].startswith('+') and args.spec[0].endswith('.py'): # Hack: Special case for defining a spec in Python for testing purposes # Use this if you want to define a Stone spec using a Python module. # The module should should contain an api variable that references a # :class:`stone.api.Api` object. try: api = imp.load_source('api', args.api[0]).api # pylint: disable=redefined-outer-name except ImportError as e: print('error: Could not import API description due to:', e, file=sys.stderr) sys.exit(1) else: if args.spec: specs = [] read_from_stdin = False for spec_path in args.spec: if spec_path == '-': read_from_stdin = True elif not spec_path.endswith('.stone'): print("error: Specification '%s' must have a .stone extension." % spec_path, file=sys.stderr) sys.exit(1) elif not os.path.exists(spec_path): print("error: Specification '%s' cannot be found." % spec_path, file=sys.stderr) sys.exit(1) else: with open(spec_path, encoding='utf-8') as f: specs.append((spec_path, f.read())) if read_from_stdin and specs: print("error: Do not specify stdin and specification files " "simultaneously.", file=sys.stderr) sys.exit(1) if not args.spec or read_from_stdin: specs = [] if debug: print('Reading specification from stdin.') if six.PY2: UTF8Reader = codecs.getreader('utf8') sys.stdin = UTF8Reader(sys.stdin) stdin_text = sys.stdin.read() else: stdin_buffer = sys.stdin.buffer # pylint: disable=no-member,useless-suppression stdin_text = io.TextIOWrapper(stdin_buffer, encoding='utf-8').read() parts = stdin_text.split('namespace') if len(parts) == 1: specs.append(('stdin.1', parts[0])) else: specs.append( ('stdin.1', '%snamespace%s' % (parts.pop(0), parts.pop(0)))) while parts: specs.append(('stdin.%s' % (len(specs) + 1), 'namespace%s' % parts.pop(0))) if args.filter_by_route_attr: route_filter, route_filter_errors = parse_route_attr_filter( args.filter_by_route_attr, debug) if route_filter_errors: print('Error(s) in route filter:', file=sys.stderr) for err in route_filter_errors: print(err, file=sys.stderr) sys.exit(1) else: route_filter = None if args.route_whitelist_filter: with open(args.route_whitelist_filter, encoding='utf-8') as f: route_whitelist_filter = json.loads(f.read()) else: route_whitelist_filter = None try: # TODO: Needs version api = specs_to_ir(specs, debug=debug, route_whitelist_filter=route_whitelist_filter) except InvalidSpec as e: print('%s:%s: error: %s' % (e.path, e.lineno, e.msg), file=sys.stderr) if debug: print('A traceback is included below in case this is a bug in ' 'Stone.\n', traceback.format_exc(), file=sys.stderr) sys.exit(1) if api is None: print('You must fix the above parsing errors for generation to ' 'continue.', file=sys.stderr) sys.exit(1) if args.whitelist_namespace_routes: for namespace_name in args.whitelist_namespace_routes: if namespace_name not in api.namespaces: print('error: Whitelisted namespace missing from spec: %s' % namespace_name, file=sys.stderr) sys.exit(1) for namespace in api.namespaces.values(): if namespace.name not in args.whitelist_namespace_routes: namespace.routes = [] namespace.route_by_name = {} namespace.routes_by_name = {} if args.blacklist_namespace_routes: for namespace_name in args.blacklist_namespace_routes: if namespace_name not in api.namespaces: print('error: Blacklisted namespace missing from spec: %s' % namespace_name, file=sys.stderr) sys.exit(1) else: namespace = api.namespaces[namespace_name] namespace.routes = [] namespace.route_by_name = {} namespace.routes_by_name = {} if route_filter: for namespace in api.namespaces.values(): filtered_routes = [] for route in namespace.routes: if route_filter.eval(route): filtered_routes.append(route) namespace.routes = [] namespace.route_by_name = {} namespace.routes_by_name = {} for route in filtered_routes: namespace.add_route(route) if args.attribute: attrs = set(args.attribute) if ':all' in attrs: attrs = {field.name for field in api.route_schema.fields} else: attrs = set() for namespace in api.namespaces.values(): for route in namespace.routes: for k in list(route.attrs.keys()): if k not in attrs: del route.attrs[k] # Remove attrs that weren't specified from the route schema for field in api.route_schema.fields[:]: if field.name not in attrs: api.route_schema.fields.remove(field) del api.route_schema._fields_by_name[field.name] else: attrs.remove(field.name) # Error if specified attr isn't even a field in the route schema if attrs: attr = attrs.pop() print('error: Attribute not defined in stone_cfg.Route: %s' % attr, file=sys.stderr) sys.exit(1) if args.backend in _builtin_backends: backend_module = __import__( 'stone.backends.%s' % args.backend, fromlist=['']) elif not os.path.exists(args.backend): print("error: Backend '%s' cannot be found." % args.backend, file=sys.stderr) sys.exit(1) elif not os.path.isfile(args.backend): print("error: Backend '%s' must be a file." % args.backend, file=sys.stderr) sys.exit(1) elif not Compiler.is_stone_backend(args.backend): print("error: Backend '%s' must have a .stoneg.py extension." % args.backend, file=sys.stderr) sys.exit(1) else: # A bit hacky, but we add the folder that the backend is in to our # python path to support the case where the backend imports other # files in its local directory. new_python_path = os.path.dirname(args.backend) if new_python_path not in sys.path: sys.path.append(new_python_path) try: backend_module = imp.load_source('user_backend', args.backend) except Exception: print("error: Importing backend '%s' module raised an exception:" % args.backend, file=sys.stderr) raise c = Compiler( api, backend_module, backend_args, args.output, clean_build=args.clean_build, ) try: c.build() except BackendException as e: print('%s: error: %s raised an exception:\n%s' % (args.backend, e.backend_name, e.traceback), file=sys.stderr) sys.exit(1) if not sys.argv[0].endswith('stone'): # If we aren't running from an entry_point, then return api to make it # easier to do debugging. return api if __name__ == '__main__': # Assign api variable for easy debugging from a Python console api = main() stone-3.3.1/stone/cli_helpers.py000066400000000000000000000127521417406541500166570ustar00rootroot00000000000000import abc import six from ply import lex, yacc _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class FilterExprLexer(object): tokens = ( 'ID', 'LPAR', 'RPAR', ) # type: typing.Tuple[str, ...] # Conjunctions tokens += ( 'AND', 'OR', ) # Comparison operators tokens += ( 'NEQ', 'EQ', ) # Primitive types tokens += ( 'BOOLEAN', 'FLOAT', 'INTEGER', 'NULL', 'STRING', ) t_LPAR = r'\(' t_RPAR = r'\)' t_NEQ = r'!=' t_EQ = r'=' t_ignore = ' ' KEYWORDS = { 'and': 'AND', 'or': 'OR', } def __init__(self, debug=False): self.lexer = lex.lex(module=self, debug=debug) self.errors = [] def get_yacc_compat_lexer(self): return self.lexer def t_BOOLEAN(self, token): r'\btrue\b|\bfalse\b' token.value = (token.value == 'true') return token def t_NULL(self, token): r'\bnull\b' token.value = None return token def t_FLOAT(self, token): r'-?\d+(\.\d*(e-?\d+)?|e-?\d+)' token.value = float(token.value) return token def t_INTEGER(self, token): r'-?\d+' token.value = int(token.value) return token def t_STRING(self, token): r'\"([^\\"]|(\\.))*\"' token.value = token.value[1:-1] return token def t_ID(self, token): r'[a-zA-Z_][a-zA-Z0-9_-]*' if token.value in self.KEYWORDS: token.type = self.KEYWORDS[token.value] return token else: return token # Error handling rule def t_error(self, token): self.errors.append( ('Illegal character %s.' % repr(token.value[0]).lstrip('u'))) token.lexer.skip(1) # Test output def test(self, data): self.lexer.input(data) while True: tok = self.lexer.token() if not tok: break print(tok) class FilterExprParser(object): # Ply parser requiment: Tokens must be re-specified in parser tokens = FilterExprLexer.tokens # Ply wants a 'str' instance; this makes it work in Python 2 and 3 start = str('expr') # To match most languages, give logical conjunctions a higher precedence # than logical disjunctions. precedence = ( ('left', 'OR'), ('left', 'AND'), ) def __init__(self, debug=False): self.debug = debug self.yacc = yacc.yacc(module=self, debug=debug, write_tables=debug) self.lexer = FilterExprLexer(debug) self.errors = [] def parse(self, data): """ Args: data (str): Raw filter expression. """ parsed_data = self.yacc.parse( data, lexer=self.lexer.get_yacc_compat_lexer(), debug=self.debug) self.errors = self.lexer.errors + self.errors return parsed_data, self.errors def p_expr(self, p): 'expr : pred' p[0] = p[1] def p_expr_parens(self, p): 'expr : LPAR expr RPAR' p[0] = p[2] def p_expr_group(self, p): """expr : expr OR expr | expr AND expr""" p[0] = FilterExprConjunction(p[2], p[1], p[3]) def p_pred(self, p): 'pred : ID op primitive' p[0] = FilterExprPredicate(p[2], p[1], p[3]) def p_op(self, p): """op : NEQ | EQ""" p[0] = p[1] def p_primitive(self, p): """primitive : BOOLEAN | FLOAT | INTEGER | NULL | STRING""" p[0] = p[1] def p_error(self, token): if token: self.errors.append( ("Unexpected %s with value %s." % (token.type, repr(token.value).lstrip('u')))) else: self.errors.append('Unexpected end of expression.') class FilterExpr(object): __metaclass__ = abc.ABCMeta @abc.abstractmethod def eval(self, route): pass class FilterExprConjunction(object): def __init__(self, conj, lhs, rhs): self.conj = conj self.lhs = lhs self.rhs = rhs def eval(self, route): if self.conj == 'and': return self.lhs.eval(route) and self.rhs.eval(route) elif self.conj == 'or': return self.lhs.eval(route) or self.rhs.eval(route) else: assert False def __repr__(self): return 'EvalConj(%r, %r, %r)' % (self.conj, self.lhs, self.rhs) class FilterExprPredicate(object): def __init__(self, op, lhs, rhs): self.op = op self.lhs = lhs self.rhs = rhs def eval(self, route): val = route.attrs.get(self.lhs, None) if self.op == '=': return val == self.rhs elif self.op == '!=': return val != self.rhs else: assert False def __repr__(self): return 'EvalPred(%r, %r, %r)' % (self.op, self.lhs, self.rhs) def parse_route_attr_filter(route_attr_filter, debug=False): """ Args: route_attr_filter (str): The raw command-line input of the route filter. Returns: Tuple[FilterExpr, List[str]]: The second element is a list of errors. """ assert isinstance(route_attr_filter, six.text_type), type(route_attr_filter) parser = FilterExprParser(debug) return parser.parse(route_attr_filter) stone-3.3.1/stone/compiler.py000066400000000000000000000106161417406541500161750ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import logging import inspect import os import shutil import traceback from stone.backend import ( Backend, remove_aliases_from_api, ) class BackendException(Exception): """Saves the traceback of an exception raised by a backend.""" def __init__(self, backend_name, tb): """ :type backend_name: str :type tb: str """ super(BackendException, self).__init__() self.backend_name = backend_name self.traceback = tb class Compiler(object): """ Applies a collection of backends found in a single backend module to an API specification. """ backend_extension = '.stoneg' def __init__(self, api, backend_module, backend_args, build_path, clean_build=False): """ Creates a Compiler. :param stone.ir.Api api: A Stone description of the API. :param backend_module: Python module that contains at least one top-level class definition that descends from a :class:`stone.backend.Backend`. :param list(str) backend_args: A list of command-line arguments to pass to the backend. :param str build_path: Location to save compiled sources to. If None, source files are compiled into the same directories. :param bool clean_build: If True, the build_path is removed before source files are compiled into them. """ self._logger = logging.getLogger('stone.compiler') self.api = api self.backend_module = backend_module self.backend_args = backend_args self.build_path = build_path # Remove existing build directory if it's a clean build if clean_build and os.path.exists(self.build_path): logging.info('Cleaning existing build directory %s...', self.build_path) shutil.rmtree(self.build_path) def build(self): """Creates outputs. Outputs are files made by a backend.""" if os.path.exists(self.build_path) and not os.path.isdir(self.build_path): self._logger.error('Output path must be a folder if it already exists') return Compiler._mkdir(self.build_path) self._execute_backend_on_spec() @staticmethod def _mkdir(path): """ Creates a directory at path if it doesn't exist. If it does exist, this function does nothing. Note that if path is a file, it will not be converted to a directory. """ try: os.makedirs(path) except OSError as e: if e.errno != 17: raise @classmethod def is_stone_backend(cls, path): """ Returns True if the file name matches the format of a stone backend, ie. its inner extension of "stoneg". For example: xyz.stoneg.py """ path_without_ext, _ = os.path.splitext(path) _, second_ext = os.path.splitext(path_without_ext) return second_ext == cls.backend_extension def _execute_backend_on_spec(self): """Renders a source file into its final form.""" api_no_aliases_cache = None for attr_key in dir(self.backend_module): attr_value = getattr(self.backend_module, attr_key) if (inspect.isclass(attr_value) and issubclass(attr_value, Backend) and not inspect.isabstract(attr_value)): self._logger.info('Running backend: %s', attr_value.__name__) backend = attr_value(self.build_path, self.backend_args) if backend.preserve_aliases: api = self.api else: if not api_no_aliases_cache: api_no_aliases_cache = remove_aliases_from_api(self.api) api = api_no_aliases_cache try: backend.generate(api) except Exception: # Wrap this exception so that it isn't thought of as a bug # in the stone parser, but rather a bug in the backend. # Remove the last char of the traceback b/c it's a newline. raise BackendException( attr_value.__name__, traceback.format_exc()[:-1]) stone-3.3.1/stone/frontend/000077500000000000000000000000001417406541500156245ustar00rootroot00000000000000stone-3.3.1/stone/frontend/__init__.py000066400000000000000000000000001417406541500177230ustar00rootroot00000000000000stone-3.3.1/stone/frontend/ast.py000066400000000000000000000314611417406541500167720ustar00rootroot00000000000000from collections import OrderedDict import six class ASTNode(object): def __init__(self, path, lineno, lexpos): """ Args: lineno (int): The line number where the start of this element occurs. lexpos (int): The character offset into the file where this element occurs. """ self.path = path self.lineno = lineno self.lexpos = lexpos class AstNamespace(ASTNode): def __init__(self, path, lineno, lexpos, name, doc): """ Args: name (str): The namespace of the spec. doc (Optional[str]): The docstring for this namespace. """ super(AstNamespace, self).__init__(path, lineno, lexpos) self.name = name self.doc = doc def __str__(self): return self.__repr__() def __repr__(self): return 'AstNamespace({!r})'.format(self.name) class AstImport(ASTNode): def __init__(self, path, lineno, lexpos, target): """ Args: target (str): The name of the namespace to import. """ super(AstImport, self).__init__(path, lineno, lexpos) self.target = target def __str__(self): return self.__repr__() def __repr__(self): return 'AstImport({!r})'.format(self.target) class AstAlias(ASTNode): def __init__(self, path, lineno, lexpos, name, type_ref, doc): """ Args: name (str): The name of the alias. type_ref (AstTypeRef): The data type of the field. doc (Optional[str]): Documentation string for the alias. """ super(AstAlias, self).__init__(path, lineno, lexpos) self.name = name self.type_ref = type_ref self.doc = doc self.annotations = [] def set_annotations(self, annotations): self.annotations = annotations def __repr__(self): return 'AstAlias({!r}, {!r})'.format(self.name, self.type_ref) class AstTypeDef(ASTNode): def __init__(self, path, lineno, lexpos, name, extends, doc, fields, examples): """ Args: name (str): Name assigned to the type. extends (Optional[str]); Name of the type this inherits from. doc (Optional[str]): Docstring for the type. fields (List[AstField]): Fields of a type, not including inherited ones. examples (Optional[OrderedDict[str, AstExample]]): Map from label to example. """ super(AstTypeDef, self).__init__(path, lineno, lexpos) self.name = name assert isinstance(extends, (AstTypeRef, type(None))), type(extends) self.extends = extends assert isinstance(doc, (six.text_type, type(None))) self.doc = doc assert isinstance(fields, list) self.fields = fields assert isinstance(examples, (OrderedDict, type(None))), type(examples) self.examples = examples def __str__(self): return self.__repr__() def __repr__(self): return 'AstTypeDef({!r}, {!r}, {!r})'.format( self.name, self.extends, self.fields, ) class AstStructDef(AstTypeDef): def __init__(self, path, lineno, lexpos, name, extends, doc, fields, examples, subtypes=None): """ Args: subtypes (Tuple[List[AstSubtypeField], bool]): Inner list enumerates subtypes. The bool indicates whether this struct is a catch-all. See AstTypeDef for other constructor args. """ super(AstStructDef, self).__init__( path, lineno, lexpos, name, extends, doc, fields, examples) assert isinstance(subtypes, (tuple, type(None))), type(subtypes) self.subtypes = subtypes def __repr__(self): return 'AstStructDef({!r}, {!r}, {!r})'.format( self.name, self.extends, self.fields, ) class AstStructPatch(ASTNode): def __init__(self, path, lineno, lexpos, name, fields, examples): super(AstStructPatch, self).__init__(path, lineno, lexpos) self.name = name assert isinstance(fields, list) self.fields = fields assert isinstance(examples, (OrderedDict, type(None))), type(examples) self.examples = examples def __repr__(self): return 'AstStructPatch({!r}, {!r})'.format( self.name, self.fields, ) class AstUnionDef(AstTypeDef): def __init__(self, path, lineno, lexpos, name, extends, doc, fields, examples, closed=False): """ Args: closed (bool): Set if this is a closed union. See AstTypeDef for other constructor args. """ super(AstUnionDef, self).__init__( path, lineno, lexpos, name, extends, doc, fields, examples) self.closed = closed def __repr__(self): return 'AstUnionDef({!r}, {!r}, {!r}, {!r})'.format( self.name, self.extends, self.fields, self.closed, ) class AstUnionPatch(ASTNode): def __init__(self, path, lineno, lexpos, name, fields, examples, closed): super(AstUnionPatch, self).__init__(path, lineno, lexpos) self.name = name assert isinstance(fields, list) self.fields = fields assert isinstance(examples, (OrderedDict, type(None))), type(examples) self.examples = examples self.closed = closed def __repr__(self): return 'AstUnionPatch({!r}, {!r}, {!r})'.format( self.name, self.fields, self.closed, ) class AstTypeRef(ASTNode): def __init__(self, path, lineno, lexpos, name, args, nullable, ns): """ Args: name (str): Name of the referenced type. args (tuple[list, dict]): Arguments to type. nullable (bool): Whether the type is nullable (can be null) ns (Optional[str]): Namespace that referred type is a member of. If none, then refers to the current namespace. """ super(AstTypeRef, self).__init__(path, lineno, lexpos) self.name = name self.args = args self.nullable = nullable self.ns = ns def __repr__(self): return 'AstTypeRef({!r}, {!r}, {!r}, {!r})'.format( self.name, self.args, self.nullable, self.ns, ) class AstTagRef(ASTNode): def __init__(self, path, lineno, lexpos, tag): """ Args: tag (str): Name of the referenced type. """ super(AstTagRef, self).__init__(path, lineno, lexpos) self.tag = tag def __repr__(self): return 'AstTagRef({!r})'.format( self.tag, ) class AstAnnotationRef(ASTNode): def __init__(self, path, lineno, lexpos, annotation, ns): """ Args: annotation (str): Name of the referenced annotation. """ super(AstAnnotationRef, self).__init__(path, lineno, lexpos) self.annotation = annotation self.ns = ns def __repr__(self): return 'AstAnnotationRef({!r}, {!r})'.format( self.annotation, self.ns ) class AstAnnotationDef(ASTNode): def __init__(self, path, lineno, lexpos, name, annotation_type, annotation_type_ns, args, kwargs): """ Args: name (str): Name of the defined annotation. annotation_type (str): Type of annotation to define. annotation_type_ns (Optional[str]): Namespace where the annotation type was defined. If None, current namespace or builtin. args (str): Arguments to define annotation. kwargs (str): Keyword Arguments to define annotation. """ super(AstAnnotationDef, self).__init__(path, lineno, lexpos) self.name = name self.annotation_type = annotation_type self.annotation_type_ns = annotation_type_ns self.args = args self.kwargs = kwargs def __repr__(self): return 'AstAnnotationDef({!r}, {!r}, {!r}, {!r}, {!r})'.format( self.name, self.annotation_type, self.annotation_type_ns, self.args, self.kwargs, ) class AstAnnotationTypeDef(ASTNode): def __init__(self, path, lineno, lexpos, name, doc, params): """ Args: name (str): Name of the defined annotation type. doc (str): Docstring for the defined annotation type. params (List[AstField]): Parameters that can be passed to the annotation type. """ super(AstAnnotationTypeDef, self).__init__(path, lineno, lexpos) self.name = name self.doc = doc self.params = params def __repr__(self): return 'AstAnnotationTypeDef({!r}, {!r}, {!r})'.format( self.name, self.doc, self.params, ) class AstField(ASTNode): """ Represents both a field of a struct and a field of a union. TODO(kelkabany): Split this into two different classes. """ def __init__(self, path, lineno, lexpos, name, type_ref): """ Args: name (str): The name of the field. type_ref (AstTypeRef): The data type of the field. """ super(AstField, self).__init__(path, lineno, lexpos) self.name = name self.type_ref = type_ref self.doc = None self.has_default = False self.default = None self.annotations = [] def set_doc(self, docstring): self.doc = docstring def set_default(self, default): self.has_default = True self.default = default def set_annotations(self, annotations): self.annotations = annotations def __repr__(self): return 'AstField({!r}, {!r}, {!r})'.format( self.name, self.type_ref, self.annotations, ) class AstVoidField(ASTNode): def __init__(self, path, lineno, lexpos, name): super(AstVoidField, self).__init__(path, lineno, lexpos) self.name = name self.doc = None self.annotations = [] def set_doc(self, docstring): self.doc = docstring def set_annotations(self, annotations): self.annotations = annotations def __str__(self): return self.__repr__() def __repr__(self): return 'AstVoidField({!r}, {!r})'.format( self.name, self.annotations, ) class AstSubtypeField(ASTNode): def __init__(self, path, lineno, lexpos, name, type_ref): super(AstSubtypeField, self).__init__(path, lineno, lexpos) self.name = name self.type_ref = type_ref def __repr__(self): return 'AstSubtypeField({!r}, {!r})'.format( self.name, self.type_ref, ) class AstRouteDef(ASTNode): def __init__(self, path, lineno, lexpos, name, version, deprecated, arg_type_ref, result_type_ref, error_type_ref=None): super(AstRouteDef, self).__init__(path, lineno, lexpos) self.name = name self.version = version self.deprecated = deprecated self.arg_type_ref = arg_type_ref self.result_type_ref = result_type_ref self.error_type_ref = error_type_ref self.doc = None self.attrs = {} def set_doc(self, docstring): self.doc = docstring def set_attrs(self, attrs): self.attrs = attrs class AstAttrField(ASTNode): def __init__(self, path, lineno, lexpos, name, value): super(AstAttrField, self).__init__(path, lineno, lexpos) self.name = name self.value = value def __repr__(self): return 'AstAttrField({!r}, {!r})'.format( self.name, self.value, ) class AstExample(ASTNode): def __init__(self, path, lineno, lexpos, label, text, fields): super(AstExample, self).__init__(path, lineno, lexpos) self.label = label self.text = text self.fields = fields def __repr__(self): return 'AstExample({!r}, {!r}, {!r})'.format( self.label, self.text, self.fields, ) class AstExampleField(ASTNode): def __init__(self, path, lineno, lexpos, name, value): super(AstExampleField, self).__init__(path, lineno, lexpos) self.name = name self.value = value def __repr__(self): return 'AstExampleField({!r}, {!r})'.format( self.name, self.value, ) class AstExampleRef(ASTNode): def __init__(self, path, lineno, lexpos, label): super(AstExampleRef, self).__init__(path, lineno, lexpos) self.label = label def __repr__(self): return 'AstExampleRef({!r})'.format(self.label) stone-3.3.1/stone/frontend/exception.py000066400000000000000000000015261417406541500202000ustar00rootroot00000000000000import six class InvalidSpec(Exception): """Raise this to indicate there was an error in a specification.""" def __init__(self, msg, lineno, path=None): """ Args: msg: Error message intended for the spec writer to read. lineno: The line number the error occurred on. path: Path to the spec file with the error. """ super(InvalidSpec, self).__init__() assert isinstance(msg, six.text_type), type(msg) assert isinstance(lineno, (six.integer_types, type(None))), type(lineno) self.msg = msg self.lineno = lineno self.path = path def __str__(self): return repr(self) def __repr__(self): return 'InvalidSpec({!r}, {!r}, {!r})'.format( self.msg, self.lineno, self.path, ) stone-3.3.1/stone/frontend/frontend.py000066400000000000000000000033061417406541500200170ustar00rootroot00000000000000import logging from .exception import InvalidSpec from .parser import ( ParserFactory, ) from .ir_generator import IRGenerator logger = logging.getLogger('stone.frontend.frontend') # FIXME: Version should not have a default. def specs_to_ir(specs, version='0.1b1', debug=False, route_whitelist_filter=None): """ Converts a collection of Stone specifications into the intermediate representation used by Stone backends. The process is: Lexer -> Parser -> Semantic Analyzer -> IR Generator. The code is structured as: 1. Parser (Lexer embedded within) 2. IR Generator (Semantic Analyzer embedded within) :type specs: List[Tuple[path: str, text: str]] :param specs: `path` is never accessed and is only used to report the location of a bad spec to the user. `spec` is the text contents of a spec (.stone) file. :raises: InvalidSpec :returns: stone.ir.Api """ parser_factory = ParserFactory(debug=debug) partial_asts = [] for path, text in specs: logger.info('Parsing spec %s', path) parser = parser_factory.get_parser() if debug: parser.test_lexing(text) partial_ast = parser.parse(text, path) if parser.got_errors_parsing(): # TODO(kelkabany): Show more than one error at a time. msg, lineno, path = parser.get_errors()[0] raise InvalidSpec(msg, lineno, path) elif len(partial_ast) == 0: logger.info('Empty spec: %s', path) else: partial_asts.append(partial_ast) return IRGenerator(partial_asts, version, debug=debug, route_whitelist_filter=route_whitelist_filter).generate_IR() stone-3.3.1/stone/frontend/ir_generator.py000066400000000000000000002347651417406541500206770ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from collections import defaultdict import copy import inspect import logging _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib re = importlib.import_module(str('re')) # type: typing.Any from ..ir import ( Alias, AnnotationType, AnnotationTypeParam, Api, ApiNamespace, ApiRoute, ApiRoutesByVersion, Boolean, Bytes, CustomAnnotation, DataType, Deprecated, DeprecationInfo, Float32, Float64, Int32, Int64, is_alias, is_composite_type, is_field_type, is_list_type, is_map_type, is_nullable_type, is_primitive_type, is_struct_type, is_user_defined_type, is_union_type, is_void_type, List, Map, Nullable, Omitted, Preview, ParameterError, RedactedBlot, RedactedHash, String, Struct, StructField, TagRef, Timestamp, UInt32, UInt64, Union, UnionField, UserDefined, Void, unwrap_aliases, unwrap_nullable, ) from .exception import InvalidSpec from .ast import ( AstAlias, AstAnnotationDef, AstAnnotationTypeDef, AstImport, AstNamespace, AstRouteDef, AstStructDef, AstStructPatch, AstTagRef, AstTypeDef, AstTypeRef, AstUnionDef, AstUnionPatch, AstVoidField, ) def quote(s): assert s.replace('_', '').replace('.', '').replace('/', '').isalnum(), \ 'Only use quote() with names or IDs in Stone.' return "'%s'" % s def parse_data_types_from_doc_ref(api, doc, namespace_context, ignore_missing_entries=False): """ Given a documentation string, parse it and return all references to other data types. If there are references to routes, include also the data types of those routes. Args: - api: The API containing this doc ref. - doc: The documentation string to parse. - namespace_context: The namespace name relative to this documentation. - ignore_missing_entries: If set, this will skip references to nonexistent data types instead of raising an exception. Returns: - a list of referenced data types """ output = [] data_types, routes_by_ns = parse_data_types_and_routes_from_doc_ref( api, doc, namespace_context, ignore_missing_entries=ignore_missing_entries) for d in data_types: output.append(d) for ns_name, routes in routes_by_ns.items(): try: ns = api.namespaces[ns_name] for r in routes: for d in ns.get_route_io_data_types_for_route(r): output.append(d) except KeyError: if not ignore_missing_entries: raise return output def parse_route_name_and_version(route_repr): """ Parse a route representation string and return the route name and version number. :param route_repr: Route representation string. :return: A tuple containing route name and version number. """ if ':' in route_repr: route_name, version = route_repr.split(':', 1) try: version = int(version) except ValueError: raise ValueError('Invalid route representation: {}'.format(route_repr)) else: route_name = route_repr version = 1 return route_name, version def parse_data_types_and_routes_from_doc_ref( api, doc, namespace_context, ignore_missing_entries=False ): """ Given a documentation string, parse it and return all references to other data types and routes. Args: - api: The API containing this doc ref. - doc: The documentation string to parse. - namespace_context: The namespace name relative to this documentation. - ignore_missing_entries: If set, this will skip references to nonexistent data types instead of raising an exception. Returns: - a tuple of referenced data types and routes """ assert doc is not None data_types = set() routes = defaultdict(set) for match in doc_ref_re.finditer(doc): try: tag = match.group('tag') val = match.group('val') supplied_namespace = api.namespaces[namespace_context] if tag == 'field': if '.' in val: type_name, __ = val.split('.', 1) doc_type = supplied_namespace.data_type_by_name[type_name] data_types.add(doc_type) else: pass # no action required, because we must be referencing the same object elif tag == 'route': if '.' in val: namespace_name, val = val.split('.', 1) namespace = api.namespaces[namespace_name] else: namespace = supplied_namespace try: route_name, version = parse_route_name_and_version(val) except ValueError as ex: raise KeyError(str(ex)) route = namespace.routes_by_name[route_name].at_version[version] routes[namespace.name].add(route) elif tag == 'type': if '.' in val: namespace_name, val = val.split('.', 1) doc_type = api.namespaces[namespace_name].data_type_by_name[val] data_types.add(doc_type) else: doc_type = supplied_namespace.data_type_by_name[val] data_types.add(doc_type) except KeyError: if not ignore_missing_entries: raise return data_types, routes # Patterns for references in documentation doc_ref_re = re.compile(r':(?P[A-z]+):`(?P.*?)`') doc_ref_val_re = re.compile( r'^(null|true|false|-?\d+(\.\d*)?(e-?\d+)?|"[^\\"]*")$') # Defined Annotations BUILTIN_ANNOTATION_CLASS_BY_STRING = { 'Deprecated': Deprecated, 'Omitted': Omitted, 'Preview': Preview, 'RedactedBlot': RedactedBlot, 'RedactedHash': RedactedHash, } class Environment(dict): # The default environment won't have a name set since it applies to all # namespaces. But, every time it's copied to represent the environment # of a specific namespace, a name should be set. namespace_name = None # type: typing.Optional[typing.Text] class IRGenerator(object): data_types = [ Bytes, Boolean, Float32, Float64, Int32, Int64, List, Map, String, Timestamp, UInt32, UInt64, Void, ] default_env = Environment( **{data_type.__name__: data_type for data_type in data_types}) # FIXME: Version should not have a default. def __init__(self, partial_asts, version, debug=False, route_whitelist_filter=None): """Creates a new tower of stone. :type specs: List[Tuple[path: str, text: str]] :param specs: `path` is never accessed and is only used to report the location of a bad spec to the user. `spec` is the text contents of a spec (.stone) file. """ self._partial_asts = partial_asts self._debug = debug self._logger = logging.getLogger('stone.idl') self.api = Api(version=version) # Map of namespace name (str) -> environment (dict) self._env_by_namespace = {} # Used to check for circular references. self._resolution_in_progress = set() # Set[DataType] self._item_by_canonical_name = {} self._patch_data_by_canonical_name = {} self._routes = route_whitelist_filter def generate_IR(self): """Parses the text of each spec and returns an API description. Returns None if an error was encountered during parsing.""" raw_api = [] for partial_ast in self._partial_asts: namespace_ast_node = self._extract_namespace_ast_node(partial_ast) namespace = self.api.ensure_namespace(namespace_ast_node.name) base_name = self._get_base_name(namespace.name, namespace.name) self._item_by_canonical_name[base_name] = namespace_ast_node if namespace_ast_node.doc is not None: namespace.add_doc(namespace_ast_node.doc) raw_api.append((namespace, partial_ast)) self._add_data_types_and_routes_to_api(namespace, partial_ast) self._add_imports_to_env(raw_api) self._merge_patches() self._populate_type_attributes() self._populate_field_defaults() self._populate_enumerated_subtypes() self._populate_route_attributes() self._populate_recursive_custom_annotations() self._populate_examples() self._validate_doc_refs() self._validate_annotations() if self._routes is not None: self._filter_namespaces_by_route_whitelist() self.api.normalize() return self.api def _extract_namespace_ast_node(self, desc): """ Checks that the namespace is declared first in the spec, and that only one namespace is declared. Args: desc (List[stone.stone.parser.ASTNode]): All AST nodes in a spec file in the order they were defined. Return: stone.frontend.ast.AstNamespace: The namespace AST node. """ if len(desc) == 0 or not isinstance(desc[0], AstNamespace): if self._debug: self._logger.info('Description: %r', desc) raise InvalidSpec('First declaration in a stone must be ' 'a namespace. Possibly caused by preceding ' 'errors.', desc[0].lineno, desc[0].path) for item in desc[1:]: if isinstance(item, AstNamespace): raise InvalidSpec('Only one namespace declaration per file.', item[0].lineno, item[0].path) return desc.pop(0) def _add_data_types_and_routes_to_api(self, namespace, desc): """ From the raw output of the parser, create forward references for each user-defined type (struct, union, route, and alias). Args: namespace (stone.api.Namespace): Namespace for definitions. desc (List[stone.stone.parser._Element]): All AST nodes in a spec file in the order they were defined. Should not include a namespace declaration. """ env = self._get_or_create_env(namespace.name) for item in desc: if isinstance(item, AstTypeDef): api_type = self._create_type(env, item) namespace.add_data_type(api_type) self._check_canonical_name_available(item, namespace.name) elif isinstance(item, AstStructPatch) or isinstance(item, AstUnionPatch): # Handle patches later. base_name = self._get_base_name(item.name, namespace.name) self._patch_data_by_canonical_name[base_name] = (item, namespace) elif isinstance(item, AstRouteDef): route = self._create_route(env, item) namespace.add_route(route) self._check_canonical_name_available(item, namespace.name, allow_duplicate=True) elif isinstance(item, AstImport): # Handle imports later. pass elif isinstance(item, AstAlias): alias = self._create_alias(env, item) namespace.add_alias(alias) self._check_canonical_name_available(item, namespace.name) elif isinstance(item, AstAnnotationDef): annotation = self._create_annotation(env, item) namespace.add_annotation(annotation) self._check_canonical_name_available(item, namespace.name) elif isinstance(item, AstAnnotationTypeDef): annotation_type = self._create_annotation_type(env, item) namespace.add_annotation_type(annotation_type) self._check_canonical_name_available(item, namespace.name) else: raise AssertionError('Unknown AST node type %r' % item.__class__.__name__) # TODO(peichao): the name conflict checking can be merged to _create_* functions using env. def _check_canonical_name_available(self, item, namespace_name, allow_duplicate=False): base_name = self._get_base_name(item.name, namespace_name) if base_name not in self._item_by_canonical_name: self._item_by_canonical_name[base_name] = item else: stored_item = self._item_by_canonical_name[base_name] is_conflict_between_same_type = item.__class__ == stored_item.__class__ # Allow name conflicts between items of the same type when allow_duplicate is True if not is_conflict_between_same_type or not allow_duplicate: msg = ("Name of %s '%s' conflicts with name of " "%s '%s' (%s:%s).") % ( self._get_user_friendly_item_type_as_string(item), item.name, self._get_user_friendly_item_type_as_string(stored_item), stored_item.name, stored_item.path, stored_item.lineno) raise InvalidSpec(msg, item.lineno, item.path) @classmethod def _get_user_friendly_item_type_as_string(cls, item): if isinstance(item, AstTypeDef): return 'user-defined type' elif isinstance(item, AstRouteDef): return 'route' elif isinstance(item, AstAlias): return 'alias' elif isinstance(item, AstNamespace): return 'namespace' elif isinstance(item, AstAnnotationTypeDef): return 'annotation type' else: raise AssertionError('unhandled type %r' % item) def _get_base_name(self, input_str, namespace_name): return (input_str.replace('_', '').replace('/', '').lower() + namespace_name.replace('_', '').lower()) def _add_imports_to_env(self, raw_api): """ Scans raw parser output for import declarations. Checks if the imports are valid, and then creates a reference to the namespace in the environment. Args: raw_api (Tuple[Namespace, List[stone.stone.parser._Element]]): Namespace paired with raw parser output. """ for namespace, desc in raw_api: for item in desc: if isinstance(item, AstImport): if namespace.name == item.target: raise InvalidSpec('Cannot import current namespace.', item.lineno, item.path) if item.target not in self.api.namespaces: raise InvalidSpec( 'Namespace %s is not defined in any spec.' % quote(item.target), item.lineno, item.path) env = self._get_or_create_env(namespace.name) imported_env = self._get_or_create_env(item.target) if namespace.name in imported_env: # Block circular imports. The Python backend can't # easily generate code for circular references. raise InvalidSpec( 'Circular import of namespaces %s and %s ' 'detected.' % (quote(namespace.name), quote(item.target)), item.lineno, item.path) env[item.target] = imported_env def _create_alias(self, env, item): # NOTE: I don't like supporting forward references for aliases # because it makes specs harder to read. But we have to so that if a # namespace is split across multiple files, the order they're specified # in the command line which affects alias ordering is irrelevant. if item.name in env: existing_dt = env[item.name] raise InvalidSpec( 'Symbol %s already defined (%s:%d).' % (quote(item.name), existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) namespace = self.api.ensure_namespace(env.namespace_name) alias = Alias(item.name, namespace, item) env[item.name] = alias return alias def _create_annotation(self, env, item): if item.name in env: existing_dt = env[item.name] raise InvalidSpec( 'Symbol %s already defined (%s:%d).' % (quote(item.name), existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) namespace = self.api.ensure_namespace(env.namespace_name) if item.args and item.kwargs: raise InvalidSpec( 'Annotations accept either positional or keyword arguments, not both', item.lineno, item.path, ) if ((item.annotation_type_ns is None) and (item.annotation_type in BUILTIN_ANNOTATION_CLASS_BY_STRING)): annotation_class = BUILTIN_ANNOTATION_CLASS_BY_STRING[item.annotation_type] annotation = annotation_class(item.name, namespace, item, *item.args, **item.kwargs) else: if item.annotation_type_ns is not None: namespace.add_imported_namespace( self.api.ensure_namespace(item.annotation_type_ns), imported_annotation_type=True) annotation = CustomAnnotation(item.name, namespace, item, item.annotation_type, item.annotation_type_ns, item.args, item.kwargs) env[item.name] = annotation return annotation def _create_annotation_type(self, env, item): if item.name in env: existing_dt = env[item.name] raise InvalidSpec( 'Symbol %s already defined (%s:%d).' % (quote(item.name), existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) namespace = self.api.ensure_namespace(env.namespace_name) if item.name in BUILTIN_ANNOTATION_CLASS_BY_STRING: raise InvalidSpec('Cannot redefine built-in annotation type %s.' % (quote(item.name), ), item.lineno, item.path) params = [] for param in item.params: if param.annotations: raise InvalidSpec( 'Annotations cannot be applied to parameters of annotation types', param.lineno, param.path) param_type = self._resolve_type(env, param.type_ref, True) dt, nullable_dt = unwrap_nullable(param_type) if isinstance(dt, Void): raise InvalidSpec( 'Parameter {} cannot be Void.'.format(quote(param.name)), param.lineno, param.path) if nullable_dt and param.has_default: raise InvalidSpec( 'Parameter {} cannot be a nullable type and have ' 'a default specified.'.format(quote(param.name)), param.lineno, param.path) if not is_primitive_type(dt): raise InvalidSpec( 'Parameter {} must have a primitive type (possibly ' 'nullable).'.format(quote(param.name)), param.lineno, param.path) params.append(AnnotationTypeParam(param.name, param_type, param.doc, param.has_default, param.default, param)) annotation_type = AnnotationType(item.name, namespace, item.doc, params) env[item.name] = annotation_type return annotation_type def _create_type(self, env, item): """Create a forward reference for a union or struct.""" if item.name in env: existing_dt = env[item.name] raise InvalidSpec( 'Symbol %s already defined (%s:%d).' % (quote(item.name), existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) namespace = self.api.ensure_namespace(env.namespace_name) if isinstance(item, AstStructDef): try: api_type = Struct(name=item.name, namespace=namespace, ast_node=item) except ParameterError as e: raise InvalidSpec( 'Bad declaration of %s: %s' % (quote(item.name), e.args[0]), item.lineno, item.path) elif isinstance(item, AstUnionDef): api_type = Union( name=item.name, namespace=namespace, ast_node=item, closed=item.closed) else: raise AssertionError('Unknown type definition %r' % type(item)) env[item.name] = api_type return api_type def _merge_patches(self): """Injects object patches into their original object definitions.""" for patched_item, patched_namespace in self._patch_data_by_canonical_name.values(): patched_item_base_name = self._get_base_name(patched_item.name, patched_namespace.name) if patched_item_base_name not in self._item_by_canonical_name: raise InvalidSpec('Patch {} must correspond to a pre-existing data_type.'.format( quote(patched_item.name)), patched_item.lineno, patched_item.path) existing_item = self._item_by_canonical_name[patched_item_base_name] self._check_patch_type_mismatch(patched_item, existing_item) if isinstance(patched_item, (AstStructPatch, AstUnionPatch)): self._check_field_names_unique(existing_item, patched_item) existing_item.fields += patched_item.fields self._inject_patched_examples(existing_item, patched_item) else: raise AssertionError('Unknown Patch Object Type {}'.format( patched_item.__class__.__name__)) def _check_patch_type_mismatch(self, patched_item, existing_item): """Enforces that each patch has a corresponding, already-defined data type.""" def raise_mismatch_error(patched_item, existing_item, data_type_name): error_msg = ('Type mismatch. Patch {} corresponds to pre-existing ' 'data_type {} ({}:{}) that has type other than {}.') raise InvalidSpec(error_msg.format( quote(patched_item.name), quote(existing_item.name), existing_item.path, existing_item.lineno, quote(data_type_name)), patched_item.lineno, patched_item.path) if isinstance(patched_item, AstStructPatch): if not isinstance(existing_item, AstStructDef): raise_mismatch_error(patched_item, existing_item, 'struct') elif isinstance(patched_item, AstUnionPatch): if not isinstance(existing_item, AstUnionDef): raise_mismatch_error(patched_item, existing_item, 'union') else: if existing_item.closed != patched_item.closed: raise_mismatch_error( patched_item, existing_item, 'union_closed' if existing_item.closed else 'union') else: raise AssertionError( 'Unknown Patch Object Type {}'.format(patched_item.__class__.__name__)) def _check_field_names_unique(self, existing_item, patched_item): """Enforces that patched fields don't already exist.""" existing_fields_by_name = {f.name: f for f in existing_item.fields} for patched_field in patched_item.fields: if patched_field.name in existing_fields_by_name.keys(): existing_field = existing_fields_by_name[patched_field.name] raise InvalidSpec('Patched field {} overrides pre-existing field in {} ({}:{}).' .format(quote(patched_field.name), quote(patched_item.name), existing_field.path, existing_field.lineno), patched_field.lineno, patched_field.path) def _inject_patched_examples(self, existing_item, patched_item): """Injects patched examples into original examples.""" for key, _ in patched_item.examples.items(): patched_example = patched_item.examples[key] existing_examples = existing_item.examples if key in existing_examples: existing_examples[key].fields.update(patched_example.fields) else: error_msg = 'Example defined in patch {} must correspond to a pre-existing example.' raise InvalidSpec(error_msg.format( quote(patched_item.name)), patched_example.lineno, patched_example.path) def _populate_type_attributes(self): """ Converts each struct, union, and route from a forward reference to a full definition. """ for namespace in self.api.namespaces.values(): env = self._get_or_create_env(namespace.name) # do annotations before everything else, since populating aliases # and datatypes involves setting annotations for annotation in namespace.annotations: if isinstance(annotation, CustomAnnotation): loc = annotation._ast_node.lineno, annotation._ast_node.path if annotation.annotation_type_ns: if annotation.annotation_type_ns not in env: raise InvalidSpec( 'Namespace %s is not imported' % quote(annotation.annotation_type_ns), *loc) annotation_type_env = env[annotation.annotation_type_ns] if not isinstance(annotation_type_env, Environment): raise InvalidSpec( '%s is not a namespace.' % quote(annotation.annotation_type_ns), *loc) else: annotation_type_env = env if annotation.annotation_type_name not in annotation_type_env: raise InvalidSpec( 'Annotation type %s does not exist' % quote(annotation.annotation_type_name), *loc) annotation_type = annotation_type_env[annotation.annotation_type_name] if not isinstance(annotation_type, AnnotationType): raise InvalidSpec( '%s is not an annotation type' % quote(annotation.annotation_type_name), *loc ) annotation.set_attributes(annotation_type) for alias in namespace.aliases: data_type = self._resolve_type(env, alias._ast_node.type_ref) alias.set_attributes(alias._ast_node.doc, data_type) annotations = [self._resolve_annotation_type(env, annotation) for annotation in alias._ast_node.annotations] alias.set_annotations(annotations) for data_type in namespace.data_types: if not data_type._is_forward_ref: continue self._resolution_in_progress.add(data_type) if isinstance(data_type, Struct): self._populate_struct_type_attributes(env, data_type) elif isinstance(data_type, Union): self._populate_union_type_attributes(env, data_type) else: raise AssertionError('Unhandled type: %r' % type(data_type)) self._resolution_in_progress.remove(data_type) assert len(self._resolution_in_progress) == 0 def _populate_struct_type_attributes(self, env, data_type): """ Converts a forward reference of a struct into a complete definition. """ parent_type = None extends = data_type._ast_node.extends if extends: # A parent type must be fully defined and not just a forward # reference. parent_type = self._resolve_type(env, extends, True) if isinstance(parent_type, Alias): # Restrict extending aliases because it's difficult to generate # code for it in Python. We put all type references at the end # to avoid out-of-order declaration issues, but using "extends" # in Python forces the reference to happen earlier. raise InvalidSpec( 'A struct cannot extend an alias. ' 'Use the canonical name instead.', data_type._ast_node.lineno, data_type._ast_node.path) if isinstance(parent_type, Nullable): raise InvalidSpec( 'A struct cannot extend a nullable type.', data_type._ast_node.lineno, data_type._ast_node.path) if not isinstance(parent_type, Struct): raise InvalidSpec( 'A struct can only extend another struct: ' '%s is not a struct.' % quote(parent_type.name), data_type._ast_node.lineno, data_type._ast_node.path) api_type_fields = [] for stone_field in data_type._ast_node.fields: api_type_field = self._create_struct_field(env, stone_field) api_type_fields.append(api_type_field) data_type.set_attributes( data_type._ast_node.doc, api_type_fields, parent_type) def _populate_union_type_attributes(self, env, data_type): """ Converts a forward reference of a union into a complete definition. """ parent_type = None extends = data_type._ast_node.extends if extends: # A parent type must be fully defined and not just a forward # reference. parent_type = self._resolve_type(env, extends, True) if isinstance(parent_type, Alias): raise InvalidSpec( 'A union cannot extend an alias. ' 'Use the canonical name instead.', data_type._ast_node.lineno, data_type._ast_node.path) if isinstance(parent_type, Nullable): raise InvalidSpec( 'A union cannot extend a nullable type.', data_type._ast_node.lineno, data_type._ast_node.path) if not isinstance(parent_type, Union): raise InvalidSpec( 'A union can only extend another union: ' '%s is not a union.' % quote(parent_type.name), data_type._ast_node.lineno, data_type._ast_node.path) api_type_fields = [] for stone_field in data_type._ast_node.fields: if stone_field.name == 'other': raise InvalidSpec( "Union cannot define an 'other' field because it is " "reserved as the catch-all field for open unions.", stone_field.lineno, stone_field.path) api_type_fields.append(self._create_union_field(env, stone_field)) catch_all_field = None if data_type.closed: if parent_type and not parent_type.closed: # Due to the reversed super type / child type relationship for # unions, a child type cannot be closed if its parent is open # because the parent now has an extra field that is not # recognized by the child if it were substituted in for it. raise InvalidSpec( "Union cannot be closed since parent type '%s' is open." % ( parent_type.name), data_type._ast_node.lineno, data_type._ast_node.path) else: if not parent_type or parent_type.closed: # Create a catch-all field catch_all_field = UnionField( name='other', data_type=Void(), doc=None, ast_node=data_type._ast_node, catch_all=True) api_type_fields.append(catch_all_field) data_type.set_attributes( data_type._ast_node.doc, api_type_fields, parent_type, catch_all_field) def _populate_recursive_custom_annotations(self): """ Populates custom annotations applied to fields recursively. This is done in a separate pass because it requires all fields and routes to be defined so that recursive chains can be followed accurately. """ data_types_seen = set() def recurse(data_type): # primitive types do not have annotations if not is_composite_type(data_type): return set() # if we have already analyzed data type, just return result if data_type.recursive_custom_annotations is not None: return data_type.recursive_custom_annotations # handle cycles safely (annotations will be found first time at top level) if data_type in data_types_seen: return set() data_types_seen.add(data_type) annotations = set() # collect data types from subtypes recursively if is_struct_type(data_type) or is_union_type(data_type): for field in data_type.fields: annotations.update(recurse(field.data_type)) # annotations can be defined directly on fields annotations.update([(field, annotation) for annotation in field.custom_annotations]) elif is_alias(data_type): annotations.update(recurse(data_type.data_type)) # annotations can be defined directly on aliases annotations.update([(data_type, annotation) for annotation in data_type.custom_annotations]) elif is_list_type(data_type): annotations.update(recurse(data_type.data_type)) elif is_map_type(data_type): # only map values support annotations for now annotations.update(recurse(data_type.value_data_type)) elif is_nullable_type(data_type): annotations.update(recurse(data_type.data_type)) data_type.recursive_custom_annotations = annotations return annotations for namespace in self.api.namespaces.values(): namespace_annotations = set() for data_type in namespace.data_types: namespace_annotations.update(recurse(data_type)) for alias in namespace.aliases: namespace_annotations.update(recurse(alias)) for route in namespace.routes: namespace_annotations.update(recurse(route.arg_data_type)) namespace_annotations.update(recurse(route.result_data_type)) namespace_annotations.update(recurse(route.error_data_type)) # record annotation types as dependencies of the namespace. this allows for # an optimization when processing custom annotations to ignore annotation # types that are not applied to the data type, rather than recursing into it for _, annotation in namespace_annotations: if annotation.annotation_type.namespace.name != namespace.name: namespace.add_imported_namespace( annotation.annotation_type.namespace, imported_annotation_type=True) def _populate_field_defaults(self): """ Populate the defaults of each field. This is done in a separate pass because defaults that specify a union tag require the union to have been defined. """ for namespace in self.api.namespaces.values(): for data_type in namespace.data_types: # Only struct fields can have default if not isinstance(data_type, Struct): continue for field in data_type.fields: if not field._ast_node.has_default: continue if isinstance(field._ast_node.default, AstTagRef): default_value = TagRef( field.data_type, field._ast_node.default.tag) else: default_value = field._ast_node.default if not (field._ast_node.type_ref.nullable and default_value is None): # Verify that the type of the default value is correct for this field try: if field.data_type.name in ('Float32', 'Float64'): # You can assign int to the default value of float type # However float type should always have default value in float default_value = float(default_value) field.data_type.check(default_value) except ValueError as e: raise InvalidSpec( 'Field %s has an invalid default: %s' % (quote(field._ast_node.name), e), field._ast_node.lineno, field._ast_node.path) field.set_default(default_value) def _populate_route_attributes(self): """ Converts all routes from forward references to complete definitions. """ route_schema = self._validate_stone_cfg() self.api.add_route_schema(route_schema) for namespace in self.api.namespaces.values(): env = self._get_or_create_env(namespace.name) for route in namespace.routes: self._populate_route_attributes_helper(env, route, route_schema) def _populate_route_attributes_helper(self, env, route, schema): """ Converts a single forward reference of a route into a complete definition. """ arg_dt = self._resolve_type(env, route._ast_node.arg_type_ref) result_dt = self._resolve_type(env, route._ast_node.result_type_ref) error_dt = self._resolve_type(env, route._ast_node.error_type_ref) ast_deprecated = route._ast_node.deprecated if ast_deprecated: assert ast_deprecated[0] new_route_name = ast_deprecated[1] new_route_version = ast_deprecated[2] if new_route_name: assert new_route_version is_not_defined = False is_not_route = False if new_route_name in env: if isinstance(env[new_route_name], ApiRoutesByVersion): if new_route_version not in env[new_route_name].at_version: is_not_defined = True else: is_not_route = True else: is_not_defined = True if is_not_defined: raise InvalidSpec( 'Undefined route %s at version %d.' % ( quote(new_route_name), new_route_version), route._ast_node.lineno, route._ast_node.path) if is_not_route: raise InvalidSpec( '%s must be a route.' % quote(new_route_name), route._ast_node.lineno, route._ast_node.path) new_route = env[new_route_name].at_version[new_route_version] deprecated = DeprecationInfo(new_route) else: deprecated = DeprecationInfo() else: deprecated = None attr_by_name = {} for attr in route._ast_node.attrs: attr_by_name[attr.name] = attr try: validated_attrs = schema.check_attr_repr(attr_by_name) except KeyError as e: raise InvalidSpec( "Route does not define attr key '%s'." % e.args[0], route._ast_node.lineno, route._ast_node.path) route.set_attributes( deprecated=deprecated, doc=route._ast_node.doc, arg_data_type=arg_dt, result_data_type=result_dt, error_data_type=error_dt, attrs=validated_attrs) def _create_struct_field(self, env, stone_field): """ This function resolves symbols to objects that we've instantiated in the current environment. For example, a field with data type named "String" is pointed to a String() object. The caller needs to ensure that this stone_field is for a Struct and not for a Union. Returns: stone.data_type.StructField: A field of a struct. """ if isinstance(stone_field, AstVoidField): raise InvalidSpec( 'Struct field %s cannot have a Void type.' % quote(stone_field.name), stone_field.lineno, stone_field.path) data_type = self._resolve_type(env, stone_field.type_ref) annotations = [self._resolve_annotation_type(env, annotation) for annotation in stone_field.annotations] if isinstance(data_type, Void): raise InvalidSpec( 'Struct field %s cannot have a Void type.' % quote(stone_field.name), stone_field.lineno, stone_field.path) elif isinstance(data_type, Nullable) and stone_field.has_default: raise InvalidSpec('Field %s cannot be a nullable ' 'type and have a default specified.' % quote(stone_field.name), stone_field.lineno, stone_field.path) api_type_field = StructField( name=stone_field.name, data_type=data_type, doc=stone_field.doc, ast_node=stone_field, ) api_type_field.set_annotations(annotations) return api_type_field def _create_union_field(self, env, stone_field): """ This function resolves symbols to objects that we've instantiated in the current environment. For example, a field with data type named "String" is pointed to a String() object. The caller needs to ensure that this stone_field is for a Union and not for a Struct. Returns: stone.data_type.UnionField: A field of a union. """ annotations = [self._resolve_annotation_type(env, annotation) for annotation in stone_field.annotations] if isinstance(stone_field, AstVoidField): api_type_field = UnionField( name=stone_field.name, data_type=Void(), doc=stone_field.doc, ast_node=stone_field) else: data_type = self._resolve_type(env, stone_field.type_ref) if isinstance(data_type, Void): raise InvalidSpec('Union member %s cannot have Void ' 'type explicit, omit Void instead.' % quote(stone_field.name), stone_field.lineno, stone_field.path) api_type_field = UnionField( name=stone_field.name, data_type=data_type, doc=stone_field.doc, ast_node=stone_field) api_type_field.set_annotations(annotations) return api_type_field def _instantiate_data_type(self, data_type_class, data_type_args, loc): """ Responsible for instantiating a data type with additional attributes. This method ensures that the specified attributes are valid. Args: data_type_class (DataType): The class to instantiate. data_type_attrs (dict): A map from str -> values of attributes. These will be passed into the constructor of data_type_class as keyword arguments. Returns: stone.data_type.DataType: A parameterized instance. """ assert issubclass(data_type_class, DataType), \ 'Expected stone.data_type.DataType, got %r' % data_type_class argspec = inspect.getargspec(data_type_class.__init__) # noqa: E501 # pylint: disable=deprecated-method,useless-suppression argspec.args.remove('self') num_args = len(argspec.args) # Unfortunately, argspec.defaults is None if there are no defaults num_defaults = len(argspec.defaults or ()) pos_args, kw_args = data_type_args if (num_args - num_defaults) > len(pos_args): # Report if a positional argument is missing raise InvalidSpec( 'Missing positional argument %s for %s type' % (quote(argspec.args[len(pos_args)]), quote(data_type_class.__name__)), *loc) elif (num_args - num_defaults) < len(pos_args): # Report if there are too many positional arguments raise InvalidSpec( 'Too many positional arguments for %s type' % quote(data_type_class.__name__), *loc) # Map from arg name to bool indicating whether the arg has a default args = {} for i, key in enumerate(argspec.args): args[key] = (i >= num_args - num_defaults) for key in kw_args: # Report any unknown keyword arguments if key not in args: raise InvalidSpec('Unknown argument %s to %s type.' % (quote(key), quote(data_type_class.__name__)), *loc) # Report any positional args that are defined as keywords args. if not args[key]: raise InvalidSpec( 'Positional argument %s cannot be specified as a ' 'keyword argument.' % quote(key), *loc) del args[key] try: return data_type_class(*pos_args, **kw_args) except ParameterError as e: # Each data type validates its own attributes, and will raise a # ParameterError if the type or value is bad. raise InvalidSpec('Bad argument to %s type: %s' % (quote(data_type_class.__name__), e.args[0]), *loc) def _resolve_type(self, env, type_ref, enforce_fully_defined=False): """ Resolves the data type referenced by type_ref. If `enforce_fully_defined` is True, then the referenced type must be fully populated (fields, parent_type, ...), and not simply a forward reference. """ loc = type_ref.lineno, type_ref.path orig_namespace_name = env.namespace_name if type_ref.ns: # TODO(kelkabany): If a spec file imports a namespace, it is # available to all spec files that are part of the same namespace. # Might want to introduce the concept of an environment specific # to a file. if type_ref.ns not in env: raise InvalidSpec( 'Namespace %s is not imported' % quote(type_ref.ns), *loc) env = env[type_ref.ns] if not isinstance(env, Environment): raise InvalidSpec( '%s is not a namespace.' % quote(type_ref.ns), *loc) if type_ref.name not in env: raise InvalidSpec( 'Symbol %s is undefined.' % quote(type_ref.name), *loc) obj = env[type_ref.name] if obj is Void and type_ref.nullable: raise InvalidSpec('Void cannot be marked nullable.', *loc) elif inspect.isclass(obj): resolved_data_type_args = self._resolve_args(env, type_ref.args) data_type = self._instantiate_data_type( obj, resolved_data_type_args, (type_ref.lineno, type_ref.path)) elif isinstance(obj, ApiRoutesByVersion): raise InvalidSpec('A route cannot be referenced here.', *loc) elif type_ref.args[0] or type_ref.args[1]: # An instance of a type cannot have any additional # attributes specified. raise InvalidSpec('Attributes cannot be specified for ' 'instantiated type %s.' % quote(type_ref.name), *loc) else: data_type = env[type_ref.name] if type_ref.ns: # Add the source namespace as an import. namespace = self.api.ensure_namespace(orig_namespace_name) if isinstance(data_type, UserDefined): namespace.add_imported_namespace( self.api.ensure_namespace(type_ref.ns), imported_data_type=True) elif isinstance(data_type, Alias): namespace.add_imported_namespace( self.api.ensure_namespace(type_ref.ns), imported_alias=True) if (enforce_fully_defined and isinstance(data_type, UserDefined) and data_type._is_forward_ref): if data_type in self._resolution_in_progress: raise InvalidSpec( 'Unresolvable circular reference for type %s.' % quote(type_ref.name), *loc) self._resolution_in_progress.add(data_type) if isinstance(data_type, Struct): self._populate_struct_type_attributes(env, data_type) elif isinstance(data_type, Union): self._populate_union_type_attributes(env, data_type) self._resolution_in_progress.remove(data_type) if type_ref.nullable: unwrapped_dt, _ = unwrap_aliases(data_type) if isinstance(unwrapped_dt, Nullable): raise InvalidSpec( 'Cannot mark reference to nullable type as nullable.', *loc) data_type = Nullable(data_type) return data_type def _resolve_annotation_type(self, env, annotation_ref): """ Resolves the annotation type referenced by annotation_ref. """ loc = annotation_ref.lineno, annotation_ref.path if annotation_ref.ns: if annotation_ref.ns not in env: raise InvalidSpec( 'Namespace %s is not imported' % quote(annotation_ref.ns), *loc) env = env[annotation_ref.ns] if not isinstance(env, Environment): raise InvalidSpec( '%s is not a namespace.' % quote(annotation_ref.ns), *loc) if annotation_ref.annotation not in env: raise InvalidSpec( 'Annotation %s does not exist.' % quote(annotation_ref.annotation), *loc) return env[annotation_ref.annotation] def _resolve_args(self, env, args): """ Resolves type references in data type arguments to data types in the environment. """ pos_args, kw_args = args def check_value(v): if isinstance(v, AstTypeRef): return self._resolve_type(env, v) else: return v new_pos_args = [check_value(pos_arg) for pos_arg in pos_args] new_kw_args = {k: check_value(v) for k, v in kw_args.items()} return new_pos_args, new_kw_args def _create_route(self, env, item): """ Constructs a route and adds it to the environment. Args: env (dict): The environment of defined symbols. A new key is added corresponding to the name of this new route. item (AstRouteDef): Raw route definition from the parser. Returns: stone.api.ApiRoutesByVersion: A group of fully-defined routes indexed by versions. """ if item.name in env: if isinstance(env[item.name], ApiRoutesByVersion): if item.version in env[item.name].at_version: existing_dt = env[item.name].at_version[item.version] raise InvalidSpec( 'Route %s at version %d already defined (%s:%d).' % ( quote(item.name), item.version, existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) else: existing_dt = env[item.name] raise InvalidSpec( 'Symbol %s already defined (%s:%d).' % ( quote(item.name), existing_dt._ast_node.path, existing_dt._ast_node.lineno), item.lineno, item.path) else: env[item.name] = ApiRoutesByVersion() route = ApiRoute( name=item.name, version=item.version, ast_node=item, ) env[route.name].at_version[route.version] = route return route def _get_or_create_env(self, namespace_name): # Because there might have already been a spec that was part of this # same namespace, the environment might already exist. if namespace_name in self._env_by_namespace: env = self._env_by_namespace[namespace_name] else: env = copy.copy(self.default_env) env.namespace_name = namespace_name self._env_by_namespace[namespace_name] = env return env def _populate_enumerated_subtypes(self): # Since enumerated subtypes require forward references, resolve them # now that all types are populated in the environment. for namespace in self.api.namespaces.values(): env = self._get_or_create_env(namespace.name) for data_type in namespace.data_types: if not (isinstance(data_type, Struct) and data_type._ast_node.subtypes): continue subtype_fields = [] for subtype_field in data_type._ast_node.subtypes[0]: subtype_name = subtype_field.type_ref.name lineno = subtype_field.type_ref.lineno path = subtype_field.type_ref.path if subtype_field.type_ref.name not in env: raise InvalidSpec( 'Undefined type %s.' % quote(subtype_name), lineno, path) subtype = self._resolve_type( env, subtype_field.type_ref, True) if not isinstance(subtype, Struct): raise InvalidSpec( 'Enumerated subtype %s must be a struct.' % quote(subtype_name), lineno, path) f = UnionField( subtype_field.name, subtype, None, subtype_field) subtype_fields.append(f) data_type.set_enumerated_subtypes(subtype_fields, data_type._ast_node.subtypes[1]) # In an enumerated subtypes tree, regular structs may only exist at # the leaves. In other words, no regular struct may inherit from a # regular struct. for data_type in namespace.data_types: if (not isinstance(data_type, Struct) or not data_type.has_enumerated_subtypes()): continue for subtype_field in data_type.get_enumerated_subtypes(): if (not subtype_field.data_type.has_enumerated_subtypes() and len(subtype_field.data_type.subtypes) > 0): raise InvalidSpec( "Subtype '%s' cannot be extended." % subtype_field.data_type.name, subtype_field.data_type._ast_node.lineno, subtype_field.data_type._ast_node.path) def _populate_examples(self): """Construct every possible example for every type. This is done in two passes. The first pass assigns examples to their associated types, but does not resolve references between examples for different types. This is because the referenced examples may not yet exist. The second pass resolves references. """ for namespace in self.api.namespaces.values(): for data_type in namespace.data_types: for example in data_type._ast_node.examples.values(): data_type._add_example(example) for namespace in self.api.namespaces.values(): for data_type in namespace.data_types: data_type._compute_examples() def _validate_doc_refs(self): """ Validates that all the documentation references across every docstring in every spec are formatted properly, have valid values, and make references to valid symbols. """ for namespace in self.api.namespaces.values(): env = self._get_or_create_env(namespace.name) # Validate the doc refs of each api entity that has a doc for data_type in namespace.data_types: if data_type.doc: self._validate_doc_refs_helper( env, data_type.doc, (data_type._ast_node.lineno + 1, data_type._ast_node.path), data_type) for field in data_type.fields: if field.doc: self._validate_doc_refs_helper( env, field.doc, (field._ast_node.lineno + 1, field._ast_node.path), data_type) for route in namespace.routes: if route.doc: self._validate_doc_refs_helper( env, route.doc, (route._ast_node.lineno + 1, route._ast_node.path)) def _validate_doc_refs_helper(self, env, doc, loc, type_context=None): """ Validates that all the documentation references in a docstring are formatted properly, have valid values, and make references to valid symbols. Args: env (dict): The environment of defined symbols. doc (str): The docstring to validate. lineno (int): The line number the docstring begins on in the spec. type_context (stone.data_type.UserDefined): If the docstring belongs to a user-defined type (Struct or Union) or one of its fields, set this to the type. This is needed for "field" doc refs that don't name a type to be validated. """ for match in doc_ref_re.finditer(doc): tag = match.group('tag') val = match.group('val') if tag == 'field': if '.' in val: type_name, field_name = val.split('.', 1) if type_name not in env: raise InvalidSpec( 'Bad doc reference to field %s of ' 'unknown type %s.' % (field_name, quote(type_name)), *loc) elif isinstance(env[type_name], ApiRoutesByVersion): raise InvalidSpec( 'Bad doc reference to field %s of route %s.' % (quote(field_name), quote(type_name)), *loc) if isinstance(env[type_name], Environment): # Handle reference to field in imported namespace. namespace_name, type_name, field_name = val.split('.', 2) data_type_to_check = env[namespace_name][type_name] elif isinstance(env[type_name], Alias): data_type_to_check = env[type_name].data_type else: data_type_to_check = env[type_name] if not any(field.name == field_name for field in data_type_to_check.all_fields): raise InvalidSpec( 'Bad doc reference to unknown field %s.' % quote(val), *loc) else: # Referring to a field that's a member of this type assert type_context is not None if not any(field.name == val for field in type_context.all_fields): raise InvalidSpec( 'Bad doc reference to unknown field %s.' % quote(val), *loc) elif tag == 'link': if not (1 < val.rfind(' ') < len(val) - 1): # There must be a space somewhere in the middle of the # string to separate the title from the uri. raise InvalidSpec( 'Bad doc reference to link (need a title and ' 'uri separated by a space): %s.' % quote(val), *loc) elif tag == 'route': if '.' in val: # Handle reference to route in imported namespace. namespace_name, val = val.split('.', 1) if namespace_name not in env: raise InvalidSpec( "Unknown doc reference to namespace '%s'." % namespace_name, *loc) env_to_check = env[namespace_name] else: env_to_check = env route_name, version = parse_route_name_and_version(val) if route_name not in env_to_check: raise InvalidSpec( 'Unknown doc reference to route {}.'.format(quote(route_name)), *loc) if not isinstance(env_to_check[route_name], ApiRoutesByVersion): raise InvalidSpec( 'Doc reference to type {} is not a route.'.format(quote(route_name)), *loc) if version not in env_to_check[route_name].at_version: raise InvalidSpec( 'Doc reference to route {} has undefined version {}.'.format( quote(route_name), version), *loc) elif tag == 'type': if '.' in val: # Handle reference to type in imported namespace. namespace_name, val = val.split('.', 1) if namespace_name not in env: raise InvalidSpec( "Unknown doc reference to namespace '%s'." % namespace_name, *loc) env_to_check = env[namespace_name] else: env_to_check = env if val not in env_to_check: raise InvalidSpec( "Unknown doc reference to type '%s'." % val, *loc) elif not isinstance(env_to_check[val], (Struct, Union)): raise InvalidSpec( 'Doc reference to type %s is not a struct or union.' % quote(val), *loc) elif tag == 'val': if not doc_ref_val_re.match(val): raise InvalidSpec( 'Bad doc reference value %s.' % quote(val), *loc) else: raise InvalidSpec( 'Unknown doc reference tag %s.' % quote(tag), *loc) def _validate_annotations(self): """ Validates that all annotations are attached to proper types and that no field has conflicting inherited or direct annotations. We need to go through all reference chains to make sure we don't override a redactor set on a parent alias or type """ for namespace in self.api.namespaces.values(): for data_type in namespace.data_types: for field in data_type.fields: if field.redactor: self._validate_field_can_be_tagged_with_redactor(field) for alias in namespace.aliases: if alias.redactor: self._validate_object_can_be_tagged_with_redactor(alias) def _validate_field_can_be_tagged_with_redactor(self, field): """ Validates that the field type can be annotated and that alias does not have conflicting annotations. """ if is_alias(field.data_type): raise InvalidSpec( "Redactors can only be applied to alias definitions, not " "to alias references.", field._ast_node.lineno, field._ast_node.path) self._validate_object_can_be_tagged_with_redactor(field) def _validate_object_can_be_tagged_with_redactor(self, annotated_object): """ Validates that the object type can be annotated and object does not have conflicting annotations. """ data_type = annotated_object.data_type name = annotated_object.name loc = annotated_object._ast_node.lineno, annotated_object._ast_node.path curr_data_type = data_type while isinstance(curr_data_type, Alias) or isinstance(curr_data_type, Nullable): # aliases have redactors assocaited with the type itself if hasattr(curr_data_type, 'redactor') and curr_data_type.redactor: raise InvalidSpec("A redactor has already been defined for '%s' by '%s'." % (str(name), str(curr_data_type.name)), *loc) curr_data_type = curr_data_type.data_type if hasattr(annotated_object, 'redactor') and annotated_object.redactor: if is_map_type(curr_data_type) or is_list_type(curr_data_type): while True: if is_map_type(curr_data_type): curr_data_type = curr_data_type.value_data_type else: curr_data_type = curr_data_type.data_type should_continue = (is_map_type(curr_data_type) or is_list_type(curr_data_type) or is_nullable_type(curr_data_type)) if should_continue is False: break if is_user_defined_type(curr_data_type) or is_void_type(curr_data_type): raise InvalidSpec("Redactors can't be applied to user-defined or void types.", *loc) def _validate_stone_cfg(self): """ Returns: Struct: A schema for route attributes. """ def mk_route_schema(): s = Struct('Route', ApiNamespace('stone_cfg'), None) s.set_attributes(None, [], None) return s try: stone_cfg = self.api.namespaces.pop('stone_cfg') except KeyError: return mk_route_schema() if stone_cfg.routes: route = stone_cfg.routes[0] raise InvalidSpec( 'No routes can be defined in the stone_cfg namespace.', route._ast_node.lineno, route._ast_node.path, ) if not stone_cfg.data_types: return mk_route_schema() for data_type in stone_cfg.data_types: if data_type.name != 'Route': raise InvalidSpec( "Only a struct named 'Route' can be defined in the " "stone_cfg namespace.", data_type._ast_node.lineno, data_type._ast_node.path, ) # TODO: are we always guaranteed at least one data type? # pylint: disable=undefined-loop-variable return data_type def _filter_namespaces_by_route_whitelist(self): """ Given a parsed API in IR form, filter the user-defined datatypes so that they include only the route datatypes and their direct dependencies. """ assert self._routes is not None, "Missing route whitelist" assert 'route_whitelist' in self._routes assert 'datatype_whitelist' in self._routes # Get route whitelist in canonical form route_whitelist = {} for namespace_name, route_reprs in self._routes['route_whitelist'].items(): new_route_reprs = [] if route_reprs == ['*']: namespace = self.api.namespaces[namespace_name] new_route_reprs = [route.name_with_version() for route in namespace.routes] else: for route_repr in route_reprs: route_name, version = parse_route_name_and_version(route_repr) if version > 1: new_route_reprs.append('{}:{}'.format(route_name, version)) else: new_route_reprs.append(route_name) route_whitelist[namespace_name] = new_route_reprs # Parse the route whitelist and populate any starting data types route_data_types = [] for namespace_name, route_reprs in route_whitelist.items(): # Error out if user supplied nonexistent namespace if namespace_name not in self.api.namespaces: raise AssertionError('Namespace %s is not defined!' % namespace_name) namespace = self.api.namespaces[namespace_name] # Parse namespace doc refs and add them to the starting data types if namespace.doc is not None: route_data_types.extend( parse_data_types_from_doc_ref(self.api, namespace.doc, namespace_name)) # Parse user-specified routes and add them to the starting data types # Note that this may add duplicates, but that's okay, as the recursion # keeps track of visited data types. assert '*' not in route_reprs for routes_repr in route_reprs: route_name, version = parse_route_name_and_version(routes_repr) if route_name not in namespace.routes_by_name or \ version not in namespace.routes_by_name[route_name].at_version: raise AssertionError('Route %s at version %d is not defined!' % (route_name, version)) route = namespace.routes_by_name[route_name].at_version[version] route_data_types.extend(namespace.get_route_io_data_types_for_route(route)) if route.doc is not None: route_data_types.extend( parse_data_types_from_doc_ref(self.api, route.doc, namespace_name)) # Parse the datatype whitelist and populate any starting data types for namespace_name, datatype_names in self._routes['datatype_whitelist'].items(): if namespace_name not in self.api.namespaces: raise AssertionError('Namespace %s is not defined!' % namespace_name) # Parse namespace doc refs and add them to the starting data types namespace = self.api.namespaces[namespace_name] if namespace.doc is not None: route_data_types.extend( parse_data_types_from_doc_ref(self.api, namespace.doc, namespace_name)) for datatype_name in datatype_names: if datatype_name not in self.api.namespaces[namespace_name].data_type_by_name: raise AssertionError('Datatype %s is not defined!' % datatype_name) data_type = self.api.namespaces[namespace_name].data_type_by_name[datatype_name] route_data_types.append(data_type) # Recurse on dependencies output_types_by_ns, output_routes_by_ns = self._find_dependencies(route_data_types) # Update the IR representation. This involves editing the data types and # routes for each namespace. for namespace in self.api.namespaces.values(): data_types = list(set(output_types_by_ns[namespace.name])) # defaults to empty list namespace.data_types = data_types namespace.data_type_by_name = {d.name: d for d in data_types} output_route_reprs = [output_route.name_with_version() for output_route in output_routes_by_ns[namespace.name]] if namespace.name in route_whitelist: whitelisted_route_reprs = route_whitelist[namespace.name] route_reprs = list(set(whitelisted_route_reprs + output_route_reprs)) else: route_reprs = output_route_reprs routes = [] for route_repr in route_reprs: route_name, version = parse_route_name_and_version(route_repr) route = namespace.routes_by_name[route_name].at_version[version] routes.append(route) namespace.routes = [] namespace.route_by_name = {} namespace.routes_by_name = {} # We need a stable sort in order to keep the resultant output # the same across runs. routes.sort() for route in routes: namespace.add_route(route) def _find_dependencies(self, data_types): output_types = defaultdict(list) output_routes = defaultdict(set) seen = set() for t in data_types: self._find_dependencies_recursive(t, seen, output_types, output_routes) return output_types, output_routes def _find_dependencies_recursive(self, data_type, seen, output_types, output_routes, type_context=None): # Define a function that recursively traverses data types and populates # the data structures defined above. if data_type in seen: # if we've visited a data type already, no need to revisit return elif is_primitive_type(data_type): # primitive types represent leaf nodes in the tree return elif is_struct_type(data_type) or is_union_type(data_type): # recurse on fields and parent types for structs and unions # also recurse on enumerated subtypes for structs if present seen.add(data_type) output_types[data_type.namespace.name].append(data_type) for field in data_type.all_fields: self._find_dependencies_recursive(field, seen, output_types, output_routes, type_context=data_type) if data_type.parent_type is not None: self._find_dependencies_recursive(data_type.parent_type, seen, output_types, output_routes) if data_type.doc is not None: doc_types, routes_by_ns = parse_data_types_and_routes_from_doc_ref( self.api, data_type.doc, data_type.namespace.name) for t in doc_types: self._find_dependencies_recursive(t, seen, output_types, output_routes) for namespace_name, routes in routes_by_ns.items(): route_namespace = self.api.namespaces[namespace_name] for route in routes: output_routes[namespace_name].add(route) route_types = route_namespace.get_route_io_data_types_for_route(route) for route_type in route_types: self._find_dependencies_recursive(route_type, seen, output_types, output_routes) if is_struct_type(data_type) and data_type.has_enumerated_subtypes(): for subtype in data_type.get_enumerated_subtypes(): self._find_dependencies_recursive(subtype, seen, output_types, output_routes, type_context=data_type) elif is_alias(data_type) or is_field_type(data_type): assert (is_field_type(data_type)) == (type_context is not None) if is_alias(data_type): namespace_context = data_type.namespace.name else: namespace_context = type_context.namespace.name seen.add(data_type) self._find_dependencies_recursive(data_type.data_type, seen, output_types, output_routes) if data_type.doc is not None: doc_types, routes_by_ns = parse_data_types_and_routes_from_doc_ref( self.api, data_type.doc, namespace_context) for t in doc_types: self._find_dependencies_recursive(t, seen, output_types, output_routes) for namespace_name, routes in routes_by_ns.items(): route_namespace = self.api.namespaces[namespace_name] for route in routes: output_routes[namespace_name].add(route) route_types = route_namespace.get_route_io_data_types_for_route(route) for route_type in route_types: self._find_dependencies_recursive(route_type, seen, output_types, output_routes) elif is_list_type(data_type) or is_nullable_type(data_type): # recurse on underlying field for aliases, lists, nullables, and fields seen.add(data_type) self._find_dependencies_recursive(data_type.data_type, seen, output_types, output_routes) elif is_map_type(data_type): # recurse on key/value fields for maps seen.add(data_type) self._find_dependencies_recursive(data_type.key_data_type, seen, output_types, output_routes) self._find_dependencies_recursive(data_type.value_data_type, seen, output_types, output_routes) else: assert False, "Unexpected type in: %s" % data_type stone-3.3.1/stone/frontend/lexer.py000066400000000000000000000342471417406541500173270ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import logging import os import ply.lex as lex _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class MultiToken(object): """Object used to monkeypatch ply.lex so that we can return multiple tokens from one lex operation.""" def __init__(self, tokens): self.type = tokens[0].type self.tokens = tokens # Represents a null value. We want to differentiate between the Python "None" # and null in several places. NullToken = object() class Lexer(object): """ Lexer. Tokenizes stone files. """ states = ( ('WSIGNORE', 'inclusive'), ) def __init__(self): self.lex = None self.tokens_queue = None # The current indentation "level" rather than a count of spaces. self.cur_indent = None self._logger = logging.getLogger('stone.stone.lexer') self.last_token = None # [(character, line number), ...] self.errors = [] def input(self, file_data, **kwargs): """ Required by ply.yacc for this to quack (duck typing) like a ply lexer. :param str file_data: Contents of the file to lex. """ self.lex = lex.lex(module=self, **kwargs) self.tokens_queue = [] self.cur_indent = 0 # Hack to avoid tokenization bugs caused by files that do not end in a # new line. self.lex.input(file_data + '\n') def token(self): """ Returns the next LexToken. Returns None when all tokens have been exhausted. """ if self.tokens_queue: self.last_token = self.tokens_queue.pop(0) else: r = self.lex.token() if isinstance(r, MultiToken): self.tokens_queue.extend(r.tokens) self.last_token = self.tokens_queue.pop(0) else: if r is None and self.cur_indent > 0: if (self.last_token and self.last_token.type not in ('NEWLINE', 'LINE')): newline_token = _create_token( 'NEWLINE', '\n', self.lex.lineno, self.lex.lexpos) self.tokens_queue.append(newline_token) dedent_count = self.cur_indent dedent_token = _create_token( 'DEDENT', '\t', self.lex.lineno, self.lex.lexpos) self.tokens_queue.extend([dedent_token] * dedent_count) self.cur_indent = 0 self.last_token = self.tokens_queue.pop(0) else: self.last_token = r return self.last_token def test(self, data): """Logs all tokens for human inspection. Useful for debugging.""" self.input(data) while True: token = self.token() if not token: break self._logger.debug('Token %r', token) # List of token names tokens = ( 'ID', 'KEYWORD', 'PATH', 'DOT', ) # type: typing.Tuple[typing.Text, ...] # Whitespace tokens tokens += ( 'DEDENT', 'INDENT', 'NEWLINE', ) # Attribute lists, aliases tokens += ( 'COMMA', 'EQ', 'LPAR', 'RPAR', ) # Primitive types tokens += ( 'BOOLEAN', 'FLOAT', 'INTEGER', 'NULL', 'STRING', ) # List notation tokens += ( 'LBRACKET', 'RBRACKET', ) # Map notation tokens += ( 'LBRACE', 'RBRACE', 'COLON', ) tokens += ( 'Q', ) # Annotation notation tokens += ( 'AT', ) # Regular expression rules for simple tokens t_DOT = r'\.' t_LBRACKET = r'\[' t_RBRACKET = r'\]' t_EQ = r'=' t_COMMA = r',' t_Q = r'\?' t_LBRACE = r'\{' t_RBRACE = r'\}' t_COLON = r'\:' t_AT = r'@' # TODO(kelkabany): Use scoped/conditional lexing to restrict where keywords # are identified as such. KEYWORDS = [ 'alias', 'annotation', 'annotation_type', 'attrs', 'by', 'deprecated', 'doc', 'example', 'error', 'extends', 'import', 'namespace', 'patch', 'route', 'struct', 'union', 'union_closed', ] RESERVED = { 'annotation': 'ANNOTATION', 'annotation_type': 'ANNOTATION_TYPE', 'attrs': 'ATTRS', 'deprecated': 'DEPRECATED', 'by': 'BY', 'extends': 'EXTENDS', 'import': 'IMPORT', 'patch': 'PATCH', 'route': 'ROUTE', 'struct': 'STRUCT', 'union': 'UNION', 'union_closed': 'UNION_CLOSED', } tokens += tuple(RESERVED.values()) def t_LPAR(self, token): r'\(' token.lexer.push_state('WSIGNORE') return token def t_RPAR(self, token): r'\)' token.lexer.pop_state() return token def t_ANY_BOOLEAN(self, token): r'\btrue\b|\bfalse\b' token.value = (token.value == 'true') return token def t_ANY_NULL(self, token): r'\bnull\b' token.value = NullToken return token # No leading digits def t_ANY_ID(self, token): r'[a-zA-Z_][a-zA-Z0-9_-]*' if token.value in self.KEYWORDS: if (token.value == 'annotation_type') and self.cur_indent: # annotation_type was added as a reserved keyword relatively # late, when there could be identifers with the same name # in existing specs. because annotation_type-the-keyword can # only be used at the beginning of a non-indented line, this # check lets both the keyword and the identifer coexist and # maintains backward compatibility. # Note: this is kind of a hack, and we should get rid of it if # the lexer gets better at telling keywords from identifiers in general. return token token.type = self.RESERVED.get(token.value, 'KEYWORD') return token else: return token def t_ANY_PATH(self, token): r'\/[/a-zA-Z0-9_-]*' return token def t_ANY_FLOAT(self, token): r'-?\d+(\.\d*(e-?\d+)?|e-?\d+)' token.value = float(token.value) return token def t_ANY_INTEGER(self, token): r'-?\d+' token.value = int(token.value) return token # Read in a string while respecting the following escape sequences: # \", \\, \n, and \t. def t_ANY_STRING(self, t): r'\"([^\\"]|(\\.))*\"' escaped = 0 t.lexer.lineno += t.value.count('\n') s = t.value[1:-1] new_str = "" for i in range(0, len(s)): c = s[i] if escaped: if c == 'n': c = '\n' elif c == 't': c = '\t' new_str += c escaped = 0 else: if c == '\\': escaped = 1 else: new_str += c # remove current indentation indentation_str = ' ' * _indent_level_to_spaces_count(self.cur_indent) lines_without_indentation = [ line.replace(indentation_str, '', 1) for line in new_str.splitlines()] t.value = '\n'.join(lines_without_indentation) return t # Ignore comments. # There are two types of comments. # 1. Comments that take up a full line. These lines are ignored entirely. # 2. Comments that come after tokens in the same line. These comments # are ignored, but, we still need to emit a NEWLINE since this rule # takes all trailing newlines. # Regardless of comment type, the following line must be checked for a # DEDENT or INDENT. def t_INITIAL_comment(self, token): r'[#][^\n]*\n+' token.lexer.lineno += token.value.count('\n') # Scan backwards from the comment hash to figure out which type of # comment this is. If we find an non-ws character, we know it was a # partial line. But, if we find a newline before a non-ws character, # then we know the entire line was a comment. i = token.lexpos - 1 while i >= 0: is_full_line_comment = token.lexer.lexdata[i] == '\n' is_partial_line_comment = (not is_full_line_comment and token.lexer.lexdata[i] != ' ') if is_full_line_comment or is_partial_line_comment: newline_token = _create_token('NEWLINE', '\n', token.lineno, token.lexpos + len(token.value) - 1) newline_token.lexer = token.lexer dent_tokens = self._create_tokens_for_next_line_dent( newline_token) if is_full_line_comment: # Comment takes the full line so ignore entirely. return dent_tokens elif is_partial_line_comment: # Comment is only a partial line. Preserve newline token. if dent_tokens: dent_tokens.tokens.insert(0, newline_token) return dent_tokens else: return newline_token i -= 1 def t_WSIGNORE_comment(self, token): r'[#][^\n]*\n+' token.lexer.lineno += token.value.count('\n') newline_token = _create_token('NEWLINE', '\n', token.lineno, token.lexpos + len(token.value) - 1) newline_token.lexer = token.lexer self._check_for_indent(newline_token) # Define a rule so we can track line numbers def t_INITIAL_NEWLINE(self, newline_token): r'\n+' newline_token.lexer.lineno += newline_token.value.count('\n') dent_tokens = self._create_tokens_for_next_line_dent(newline_token) if dent_tokens: dent_tokens.tokens.insert(0, newline_token) return dent_tokens else: return newline_token def t_WSIGNORE_NEWLINE(self, newline_token): r'\n+' newline_token.lexer.lineno += newline_token.value.count('\n') self._check_for_indent(newline_token) def _create_tokens_for_next_line_dent(self, newline_token): """ Starting from a newline token that isn't followed by another newline token, returns any indent or dedent tokens that immediately follow. If indentation doesn't change, returns None. """ indent_delta = self._get_next_line_indent_delta(newline_token) if indent_delta is None or indent_delta == 0: # Next line's indent isn't relevant OR there was no change in # indentation. return None dent_type = 'INDENT' if indent_delta > 0 else 'DEDENT' dent_token = _create_token( dent_type, '\t', newline_token.lineno + 1, newline_token.lexpos + len(newline_token.value)) tokens = [dent_token] * abs(indent_delta) self.cur_indent += indent_delta return MultiToken(tokens) def _check_for_indent(self, newline_token): """ Checks that the line following a newline is indented, otherwise a parsing error is generated. """ indent_delta = self._get_next_line_indent_delta(newline_token) if indent_delta is None or indent_delta == 1: # Next line's indent isn't relevant (e.g. it's a comment) OR # next line is correctly indented. return None else: self.errors.append( ('Line continuation must increment indent by 1.', newline_token.lexer.lineno)) def _get_next_line_indent_delta(self, newline_token): """ Returns the change in indentation. The return units are in indentations rather than spaces/tabs. If the next line's indent isn't relevant (e.g. it's a comment), returns None. Since the return value might be 0, the caller should explicitly check the return type, rather than rely on truthiness. """ assert newline_token.type == 'NEWLINE', \ 'Can only search for a dent starting from a newline.' next_line_pos = newline_token.lexpos + len(newline_token.value) if next_line_pos == len(newline_token.lexer.lexdata): # Reached end of file return None line = newline_token.lexer.lexdata[next_line_pos:].split(os.linesep, 1)[0] if not line: return None lstripped_line = line.lstrip() lstripped_line_length = len(lstripped_line) if lstripped_line_length == 0: # If the next line is composed of only spaces, ignore indentation. return None if lstripped_line[0] == '#': # If it's a comment line, ignore indentation. return None indent = len(line) - lstripped_line_length if indent % 4 > 0: self.errors.append( ('Indent is not divisible by 4.', newline_token.lexer.lineno)) return None indent_delta = indent - _indent_level_to_spaces_count(self.cur_indent) return indent_delta // 4 # A string containing ignored characters (spaces and tabs) t_ignore = ' \t' # Error handling rule def t_ANY_error(self, token): self._logger.debug('Illegal character %r at line %d', token.value[0], token.lexer.lineno) self.errors.append( ('Illegal character %s.' % repr(token.value[0]).lstrip('u'), token.lexer.lineno)) token.lexer.skip(1) def _create_token(token_type, value, lineno, lexpos): """ Helper for creating ply.lex.LexToken objects. Unfortunately, LexToken does not have a constructor defined to make settings these values easy. """ token = lex.LexToken() token.type = token_type token.value = value token.lineno = lineno token.lexpos = lexpos return token def _indent_level_to_spaces_count(indent): return indent * 4 stone-3.3.1/stone/frontend/parser.py000066400000000000000000000657621417406541500175120ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict import logging import ply.yacc as yacc from .lexer import ( Lexer, NullToken, ) from .ast import ( AstAlias, AstAnnotationDef, AstAnnotationRef, AstAnnotationTypeDef, AstAttrField, AstExample, AstExampleField, AstExampleRef, AstField, AstNamespace, AstImport, AstRouteDef, AstStructDef, AstStructPatch, AstSubtypeField, AstTagRef, AstTypeRef, AstUnionDef, AstUnionPatch, AstVoidField, ) logger = logging.getLogger(str('stone.frontend.parser')) class ParserFactory(object): """ After instantiating a ParserFactory, call get_parser() to get an object with a parse() method. It so happens that the object is also a ParserFactory. The purpose of get_parser() is to reset the internal state of the fatory. The details for why these aren't cleanly separated have to do with the inability to separate out the yacc.yacc BNF definition parser from the class methods that implement the parser handling logic. Due to how ply.yacc works, the docstring of each parser method is a BNF rule. Comments that would normally be docstrings for each parser rule method are kept before the method definition. """ # Ply parser requiment: Tokens must be re-specified in parser tokens = Lexer.tokens # Ply feature: Starting grammar rule start = str('spec') # PLY wants a 'str' instance; this makes it work in Python 2 and 3 def __init__(self, debug=False): self.debug = debug self.yacc = yacc.yacc(module=self, debug=self.debug, write_tables=self.debug) self.lexer = Lexer() # [(token type, token value, line number), ...] self.errors = [] # Path to file being parsed. This is added to each token for its # utility in error reporting. But the path is never accessed, so this # is optional. self.path = None self.anony_defs = [] self.exhausted = True def get_parser(self): """ Returns a ParserFactory with the state reset so it can be used to parse again. :return: ParserFactory """ self.path = None self.anony_defs = [] self.exhausted = False return self def parse(self, data, path=None): """ Args: data (str): Raw specification text. path (Optional[str]): Path to specification on filesystem. Only used to tag tokens with the file they originated from. """ assert not self.exhausted, 'Must call get_parser() to reset state.' self.path = path parsed_data = self.yacc.parse(data, lexer=self.lexer, debug=self.debug) # It generally makes sense for lexer errors to come first, because # those can be the root of parser errors. Also, since we only show one # error max right now, it's best to show the lexing one. for err_msg, lineno in self.lexer.errors[::-1]: self.errors.insert(0, (err_msg, lineno, self.path)) parsed_data.extend(self.anony_defs) self.exhausted = True return parsed_data def test_lexing(self, data): self.lexer.test(data) def got_errors_parsing(self): """Whether the lexer or parser had errors.""" return self.errors def get_errors(self): """ If got_errors_parsing() returns True, call this to get the errors. Returns: list[tuple[msg: str, lineno: int, path: str]] """ return self.errors[:] # -------------------------------------------------------------- # Spec := Namespace Import* Definition* def p_spec_init(self, p): """spec : NL | empty""" p[0] = [] def p_spec_init_decl(self, p): """spec : namespace | import | definition""" p[0] = [p[1]] def p_spec_iter(self, p): """spec : spec namespace | spec import | spec definition""" p[0] = p[1] p[0].append(p[2]) # This covers the case where we have garbage characters in a file that # splits a NL token into two separate tokens. def p_spec_ignore_newline(self, p): 'spec : spec NL' p[0] = p[1] def p_definition(self, p): """definition : alias | annotation | annotation_type | struct | struct_patch | union | union_patch | route""" p[0] = p[1] def p_namespace(self, p): """namespace : KEYWORD ID NL | KEYWORD ID NL INDENT docsection DEDENT""" if p[1] == 'namespace': doc = None if len(p) > 4: doc = p[5] p[0] = AstNamespace( self.path, p.lineno(1), p.lexpos(1), p[2], doc) else: raise ValueError('Expected namespace keyword') def p_import(self, p): 'import : IMPORT ID NL' p[0] = AstImport(self.path, p.lineno(1), p.lexpos(1), p[2]) def p_alias(self, p): """alias : KEYWORD ID EQ type_ref NL | KEYWORD ID EQ type_ref NL INDENT annotation_ref_list docsection DEDENT""" if p[1] == 'alias': has_annotations = len(p) > 6 and p[7] is not None doc = p[8] if len(p) > 6 else None p[0] = AstAlias( self.path, p.lineno(1), p.lexpos(1), p[2], p[4], doc) if has_annotations: p[0].set_annotations(p[7]) else: raise ValueError('Expected alias keyword') def p_nl(self, p): 'NL : NEWLINE' p[0] = p[1] # Sometimes we'll have multiple consecutive newlines that the lexer has # trouble combining, so we do it in the parser. def p_nl_combine(self, p): 'NL : NL NEWLINE' p[0] = p[1] # -------------------------------------------------------------- # Primitive Types def p_primitive(self, p): """primitive : BOOLEAN | FLOAT | INTEGER | NULL | STRING""" p[0] = p[1] # -------------------------------------------------------------- # References to Types # # There are several places references to types are made: # 1. Alias sources # alias x = TypeRef # 2. Field data types # struct S # f TypeRef # 3. In arguments to type references # struct S # f TypeRef(key=TypeRef) # # A type reference can have positional and keyword arguments: # TypeRef(value1, ..., kwarg1=kwvalue1) # If it has no arguments, the parentheses can be omitted. # # If a type reference has a '?' suffix, it is a nullable type. def p_pos_arg(self, p): """pos_arg : primitive | type_ref""" p[0] = p[1] def p_pos_args_list_create(self, p): """pos_args_list : pos_arg""" p[0] = [p[1]] def p_pos_args_list_extend(self, p): """pos_args_list : pos_args_list COMMA pos_arg""" p[0] = p[1] p[0].append(p[3]) def p_kw_arg(self, p): """kw_arg : ID EQ primitive | ID EQ type_ref""" p[0] = {p[1]: p[3]} def p_kw_args(self, p): """kw_args : kw_arg""" p[0] = p[1] def p_kw_args_update(self, p): """kw_args : kw_args COMMA kw_arg""" p[0] = p[1] for key in p[3]: if key in p[1]: msg = "Keyword argument '%s' defined more than once." % key self.errors.append((msg, p.lineno(2), self.path)) p[0].update(p[3]) def p_args(self, p): """args : LPAR pos_args_list COMMA kw_args RPAR | LPAR pos_args_list RPAR | LPAR kw_args RPAR | LPAR RPAR | empty""" if len(p) > 3: if p[3] == ',': p[0] = (p[2], p[4]) elif isinstance(p[2], dict): p[0] = ([], p[2]) else: p[0] = (p[2], {}) else: p[0] = ([], {}) def p_field_nullable(self, p): """nullable : Q | empty""" p[0] = p[1] == '?' def p_type_ref(self, p): 'type_ref : ID args nullable' p[0] = AstTypeRef( path=self.path, lineno=p.lineno(1), lexpos=p.lexpos(1), name=p[1], args=p[2], nullable=p[3], ns=None, ) # A reference to a type in another namespace. def p_foreign_type_ref(self, p): 'type_ref : ID DOT ID args nullable' p[0] = AstTypeRef( path=self.path, lineno=p.lineno(1), lexpos=p.lexpos(1), name=p[3], args=p[4], nullable=p[5], ns=p[1], ) # -------------------------------------------------------------- # Annotation types # # An example annotation type: # # annotation_type Sensitive # "This is a docstring for the annotation type" # # sensitivity Int32 # # reason String? # "This is a docstring for the field" # def p_annotation_type(self, p): """annotation_type : ANNOTATION_TYPE ID NL \ INDENT docsection field_list DEDENT""" p[0] = AstAnnotationTypeDef( path=self.path, lineno=p.lineno(1), lexpos=p.lexpos(1), name=p[2], doc=p[5], params=p[6]) # -------------------------------------------------------------- # Structs # # An example struct looks as follows: # # struct S extends P # "This is a docstring for the struct" # # typed_field String # "This is a docstring for the field" # # An example struct that enumerates subtypes looks as follows: # # struct P # union # t1 S1 # t2 S2 # field String # # struct S1 extends P # ... # # struct S2 extends P # ... # def p_enumerated_subtypes(self, p): """enumerated_subtypes : uniont NL INDENT subtypes_list DEDENT | empty""" if len(p) > 2: p[0] = (p[4], p[1][0] == 'union') def p_struct(self, p): """struct : STRUCT ID inheritance NL \ INDENT docsection enumerated_subtypes field_list examples DEDENT""" self.make_struct(p) def p_anony_struct(self, p): """anony_def : STRUCT empty inheritance NL \ INDENT docsection enumerated_subtypes field_list examples DEDENT""" self.make_struct(p) def make_struct(self, p): p[0] = AstStructDef( path=self.path, lineno=p.lineno(1), lexpos=p.lexpos(1), name=p[2], extends=p[3], doc=p[6], subtypes=p[7], fields=p[8], examples=p[9]) def p_struct_patch(self, p): """struct_patch : PATCH STRUCT ID NL INDENT field_list examples DEDENT""" p[0] = AstStructPatch( path=self.path, lineno=p.lineno(1), lexpos=p.lexpos(1), name=p[3], fields=p[6], examples=p[7]) def p_inheritance(self, p): """inheritance : EXTENDS type_ref | empty""" if p[1]: if p[2].nullable: msg = 'Reference cannot be nullable.' self.errors.append((msg, p.lineno(1), self.path)) else: p[0] = p[2] def p_enumerated_subtypes_list_create(self, p): """subtypes_list : subtype_field | empty""" if p[1] is not None: p[0] = [p[1]] def p_enumerated_subtypes_list_extend(self, p): 'subtypes_list : subtypes_list subtype_field' p[0] = p[1] p[0].append(p[2]) def p_enumerated_subtype_field(self, p): 'subtype_field : ID type_ref NL' p[0] = AstSubtypeField( self.path, p.lineno(1), p.lexpos(1), p[1], p[2]) # -------------------------------------------------------------- # Fields # # Each struct has zero or more fields. A field has a name, type, # and docstring. # # TODO(kelkabany): Split fields into struct fields and union fields # since they differ in capabilities rather significantly now. def p_field_list_create(self, p): """field_list : field | empty""" if p[1] is None: p[0] = [] else: p[0] = [p[1]] def p_field_list_extend(self, p): 'field_list : field_list field' p[0] = p[1] p[0].append(p[2]) def p_default_option(self, p): """default_option : EQ primitive | EQ tag_ref | empty""" if p[1]: if isinstance(p[2], AstTagRef): p[0] = p[2] else: p[0] = p[2] def p_field(self, p): """field : ID type_ref default_option NL \ INDENT annotation_ref_list docsection anony_def_option DEDENT | ID type_ref default_option NL""" has_annotations = len(p) > 5 and p[6] is not None has_docstring = len(p) > 5 and p[7] is not None has_anony_def = len(p) > 5 and p[8] is not None p[0] = AstField( self.path, p.lineno(1), p.lexpos(1), p[1], p[2]) if p[3] is not None: if p[3] is NullToken: p[0].set_default(None) else: p[0].set_default(p[3]) if has_annotations: p[0].set_annotations(p[6]) if has_docstring: p[0].set_doc(p[7]) if has_anony_def: p[8].name = p[2].name self.anony_defs.append(p[8]) def p_anony_def_option(self, p): """anony_def_option : anony_def | empty""" p[0] = p[1] def p_tag_ref(self, p): 'tag_ref : ID' p[0] = AstTagRef(self.path, p.lineno(1), p.lexpos(1), p[1]) def p_annotation(self, p): """annotation : ANNOTATION ID EQ ID args NL | ANNOTATION ID EQ ID DOT ID args NL""" if len(p) < 8: args, kwargs = p[5] p[0] = AstAnnotationDef( self.path, p.lineno(1), p.lexpos(1), p[2], p[4], None, args, kwargs) else: args, kwargs = p[7] p[0] = AstAnnotationDef( self.path, p.lineno(1), p.lexpos(1), p[2], p[6], p[4], args, kwargs) def p_annotation_ref_list_create(self, p): """annotation_ref_list : annotation_ref | empty""" if p[1] is not None: p[0] = [p[1]] else: p[0] = None def p_annotation_ref_list_extend(self, p): """annotation_ref_list : annotation_ref_list annotation_ref""" p[0] = p[1] p[0].append(p[2]) def p_annotation_ref(self, p): """annotation_ref : AT ID NL | AT ID DOT ID NL""" if len(p) < 5: p[0] = AstAnnotationRef(self.path, p.lineno(1), p.lexpos(1), p[2], None) else: p[0] = AstAnnotationRef(self.path, p.lineno(1), p.lexpos(1), p[4], p[2]) # -------------------------------------------------------------- # Unions # # An example union looks as follows: # # union U # "This is a docstring for the union" # # void_field* # "Docstring for field with type Void" # typed_field String # # void_field demonstrates the notation for a catch all variant. def p_union(self, p): """union : uniont ID inheritance NL \ INDENT docsection field_list examples DEDENT""" self.make_union(p) def p_anony_union(self, p): """anony_def : uniont empty inheritance NL \ INDENT docsection field_list examples DEDENT""" self.make_union(p) def make_union(self, p): p[0] = AstUnionDef( path=self.path, lineno=p[1][1], lexpos=p[1][2], name=p[2], extends=p[3], doc=p[6], fields=p[7], examples=p[8], closed=p[1][0] == 'union_closed') def p_union_patch(self, p): """union_patch : PATCH uniont ID NL INDENT field_list examples DEDENT""" p[0] = AstUnionPatch( path=self.path, lineno=p[2][1], lexpos=p[2][2], name=p[3], fields=p[6], examples=p[7], closed=p[2][0] == 'union_closed') def p_uniont(self, p): """uniont : UNION | UNION_CLOSED""" p[0] = (p[1], p.lineno(1), p.lexpos(1)) def p_field_void(self, p): """field : ID NL | ID NL INDENT annotation_ref_list docsection DEDENT""" p[0] = AstVoidField(self.path, p.lineno(1), p.lexpos(1), p[1]) if len(p) > 3: if p[4] is not None: p[0].set_annotations(p[4]) if p[5] is not None: p[0].set_doc(p[5]) # -------------------------------------------------------------- # Routes # # An example route looks as follows: # # route sample-route/sub-path:2 (arg, result, error) # "This is a docstring for the route" # # attrs # key="value" # # The error type is optional. def p_route(self, p): """route : ROUTE route_name route_version route_io route_deprecation NL \ INDENT docsection attrssection DEDENT | ROUTE route_name route_version route_io route_deprecation NL""" p[0] = AstRouteDef(self.path, p.lineno(1), p.lexpos(1), p[2], p[3], p[5], *p[4]) if len(p) > 7: p[0].set_doc(p[8]) if p[9]: keys = set() for attr in p[9]: if attr.name in keys: msg = "Attribute '%s' defined more than once." % attr.name self.errors.append((msg, attr.lineno, attr.path)) keys.add(attr.name) p[0].set_attrs(p[9]) def p_route_name(self, p): 'route_name : ID route_path' if p[2]: p[0] = p[1] + p[2] else: p[0] = p[1] def p_route_path_suffix(self, p): """route_path : PATH | empty""" p[0] = p[1] def p_route_version(self, p): """route_version : COLON INTEGER | empty""" if len(p) > 2: if p[2] <= 0: msg = "Version number should be a positive integer." self.errors.append((msg, p.lineno(2), self.path)) p[0] = p[2] else: p[0] = 1 def p_route_io(self, p): """route_io : LPAR type_ref COMMA type_ref RPAR | LPAR type_ref COMMA type_ref COMMA type_ref RPAR""" if len(p) > 6: p[0] = (p[2], p[4], p[6]) else: p[0] = (p[2], p[4], None) def p_route_deprecation(self, p): """route_deprecation : DEPRECATED | DEPRECATED BY route_name route_version | empty""" if len(p) == 5: p[0] = (True, p[3], p[4]) elif p[1]: p[0] = (True, None, None) def p_attrs_section(self, p): """attrssection : ATTRS NL INDENT attr_fields DEDENT | empty""" if p[1]: p[0] = p[4] def p_attr_fields_create(self, p): 'attr_fields : attr_field' p[0] = [p[1]] def p_attr_fields_add(self, p): 'attr_fields : attr_fields attr_field' p[0] = p[1] p[0].append(p[2]) def p_attr_field(self, p): """attr_field : ID EQ primitive NL | ID EQ tag_ref NL""" if p[3] is NullToken: p[0] = AstAttrField( self.path, p.lineno(1), p.lexpos(1), p[1], None) else: p[0] = AstAttrField( self.path, p.lineno(1), p.lexpos(1), p[1], p[3]) # -------------------------------------------------------------- # Doc sections # # Doc sections appear after struct, union, and route signatures; # also after field declarations. # # They're represented by text (multi-line supported) enclosed by # quotations. # # struct S # "This is a docstring # for struct S" # # number Int64 # "This is a docstring for this field" def p_docsection(self, p): """docsection : docstring NL | empty""" if p[1] is not None: p[0] = p[1] def p_docstring_string(self, p): 'docstring : STRING' # Remove trailing whitespace on every line. p[0] = '\n'.join([line.rstrip() for line in p[1].split('\n')]) # -------------------------------------------------------------- # Examples # # Examples appear at the bottom of struct definitions to give # illustrative examples of what struct values may look like. # # struct S # number Int64 # # example default "This is a label" # number=42 def p_examples_create(self, p): """examples : example | empty""" p[0] = OrderedDict() if p[1] is not None: p[0][p[1].label] = p[1] def p_examples_add(self, p): 'examples : examples example' p[0] = p[1] if p[2].label in p[0]: existing_ex = p[0][p[2].label] self.errors.append( ("Example with label '%s' already defined on line %d." % (existing_ex.label, existing_ex.lineno), p[2].lineno, p[2].path)) p[0][p[2].label] = p[2] # It's possible for no example fields to be specified. def p_example(self, p): """example : KEYWORD ID NL INDENT docsection example_fields DEDENT | KEYWORD ID NL""" if len(p) > 4: seen_fields = set() for example_field in p[6]: if example_field.name in seen_fields: self.errors.append( ("Example with label '%s' defines field '%s' more " "than once." % (p[2], example_field.name), p.lineno(1), self.path)) seen_fields.add(example_field.name) p[0] = AstExample( self.path, p.lineno(1), p.lexpos(1), p[2], p[5], OrderedDict((f.name, f) for f in p[6])) else: p[0] = AstExample( self.path, p.lineno(1), p.lexpos(1), p[2], None, OrderedDict()) def p_example_fields_create(self, p): 'example_fields : example_field' p[0] = [p[1]] def p_example_fields_add(self, p): 'example_fields : example_fields example_field' p[0] = p[1] p[0].append(p[2]) def p_example_field(self, p): """example_field : ID EQ primitive NL | ID EQ ex_list NL | ID EQ ex_map NL""" if p[3] is NullToken: p[0] = AstExampleField( self.path, p.lineno(1), p.lexpos(1), p[1], None) else: p[0] = AstExampleField( self.path, p.lineno(1), p.lexpos(1), p[1], p[3]) def p_example_multiline(self, p): """example_field : ID EQ NL INDENT ex_map NL DEDENT""" p[0] = AstExampleField( self.path, p.lineno(1), p.lexpos(1), p[1], p[5]) def p_example_field_ref(self, p): 'example_field : ID EQ ID NL' p[0] = AstExampleField(self.path, p.lineno(1), p.lexpos(1), p[1], AstExampleRef(self.path, p.lineno(3), p.lexpos(3), p[3])) # -------------------------------------------------------------- # Example of list def p_ex_list(self, p): """ex_list : LBRACKET ex_list_items RBRACKET | LBRACKET empty RBRACKET""" if p[2] is None: p[0] = [] else: p[0] = p[2] def p_ex_list_item_primitive(self, p): 'ex_list_item : primitive' if p[1] is NullToken: p[0] = None else: p[0] = p[1] def p_ex_list_item_id(self, p): 'ex_list_item : ID' p[0] = AstExampleRef(self.path, p.lineno(1), p.lexpos(1), p[1]) def p_ex_list_item_list(self, p): 'ex_list_item : ex_list' p[0] = p[1] def p_ex_list_items_create(self, p): """ex_list_items : ex_list_item""" p[0] = [p[1]] def p_ex_list_items_extend(self, p): """ex_list_items : ex_list_items COMMA ex_list_item""" p[0] = p[1] p[0].append(p[3]) # -------------------------------------------------------------- # Maps # def p_ex_map(self, p): """ex_map : LBRACE ex_map_pairs RBRACE | LBRACE empty RBRACE""" p[0] = p[2] or {} def p_ex_map_multiline(self, p): """ex_map : LBRACE NL INDENT ex_map_pairs NL DEDENT RBRACE""" p[0] = p[4] or {} def p_ex_map_elem_primitive(self, p): """ex_map_elem : primitive""" p[0] = None if p[1] == NullToken else p[1] def p_ex_map_elem_composit(self, p): """ex_map_elem : ex_map | ex_list""" p[0] = p[1] def p_ex_map_elem_id(self, p): """ex_map_elem : ID""" p[0] = AstExampleRef(self.path, p.lineno(1), p.lexpos(1), p[1]) def p_ex_map_pair(self, p): """ex_map_pair : ex_map_elem COLON ex_map_elem""" try: p[0] = {p[1]: p[3]} except TypeError: msg = "%s is an invalid hash key because it cannot be hashed." % repr(p[1]) self.errors.append((msg, p.lineno(2), self.path)) p[0] = {} def p_ex_map_pairs_create(self, p): """ex_map_pairs : ex_map_pair """ p[0] = p[1] def p_ex_map_pairs_extend(self, p): """ex_map_pairs : ex_map_pairs COMMA ex_map_pair""" p[0] = p[1] p[0].update(p[3]) def p_ex_map_pairs_multiline(self, p): """ex_map_pairs : ex_map_pairs COMMA NL ex_map_pair""" p[0] = p[1] p[0].update(p[4]) # -------------------------------------------------------------- # In ply, this is how you define an empty rule. This is used when we want # the parser to treat a rule as optional. def p_empty(self, p): 'empty :' # Called by the parser whenever a token doesn't match any rule. def p_error(self, token): assert token is not None, "Unknown error, please report this." logger.debug('Unexpected %s(%r) at line %d', token.type, token.value, token.lineno) self.errors.append( ("Unexpected %s with value %s." % (token.type, repr(token.value).lstrip('u')), token.lineno, self.path)) stone-3.3.1/stone/ir/000077500000000000000000000000001417406541500144175ustar00rootroot00000000000000stone-3.3.1/stone/ir/__init__.py000066400000000000000000000001231417406541500165240ustar00rootroot00000000000000from .api import * # noqa: F401,F403 from .data_types import * # noqa: F401,F403 stone-3.3.1/stone/ir/api.py000066400000000000000000000440631417406541500155510ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals from collections import OrderedDict # See from distutils.version import StrictVersion import six from .data_types import ( doc_unwrap, is_alias, is_composite_type, is_list_type, is_nullable_type, ) _MYPY = False if _MYPY: import typing # pylint: disable=import-error,useless-suppression from .data_types import ( # noqa: F401 # pylint: disable=unused-import Alias, Annotation, AnnotationType, DataType, List as DataTypeList, Nullable, Struct, UserDefined, ) from stone.frontend.ast import AstRouteDef # noqa: F401 # pylint: disable=unused-import # TODO: This can be changed back to a single declaration with a # unicode literal after # makes it into a PyPi release if six.PY3: NamespaceDict = typing.Dict[typing.Text, 'ApiNamespace'] else: NamespaceDict = typing.Dict[typing.Text, b'ApiNamespace'] class Api(object): """ A full description of an API's namespaces, data types, and routes. """ def __init__(self, version): # type: (str) -> None self.version = StrictVersion(version) self.namespaces = OrderedDict() # type: NamespaceDict self.route_schema = None # type: typing.Optional[Struct] def ensure_namespace(self, name): # type: (str) -> ApiNamespace """ Only creates a namespace if it hasn't yet been defined. :param str name: Name of the namespace. :return ApiNamespace: """ if name not in self.namespaces: self.namespaces[name] = ApiNamespace(name) return self.namespaces[name] def normalize(self): # type: () -> None """ Alphabetizes namespaces and routes to make spec parsing order mostly irrelevant. """ ordered_namespaces = OrderedDict() # type: NamespaceDict # self.namespaces is currently ordered by declaration order. for namespace_name in sorted(self.namespaces.keys()): ordered_namespaces[namespace_name] = self.namespaces[namespace_name] self.namespaces = ordered_namespaces for namespace in self.namespaces.values(): namespace.normalize() def add_route_schema(self, route_schema): # type: (Struct) -> None assert self.route_schema is None self.route_schema = route_schema class _ImportReason(object): """ Tracks the reason a namespace was imported. """ def __init__(self): # type: () -> None self.alias = False self.data_type = False self.annotation = False self.annotation_type = False class ApiNamespace(object): """ Represents a category of API endpoints and their associated data types. """ def __init__(self, name): # type: (typing.Text) -> None self.name = name self.doc = None # type: typing.Optional[typing.Any] self.routes = [] # type: typing.List[ApiRoute] # TODO (peichao): route_by_name is deprecated by routes_by_name and should be removed. self.route_by_name = {} # type: typing.Dict[typing.Text, ApiRoute] self.routes_by_name = {} # type: typing.Dict[typing.Text, ApiRoutesByVersion] self.data_types = [] # type: typing.List[UserDefined] self.data_type_by_name = {} # type: typing.Dict[str, UserDefined] self.aliases = [] # type: typing.List[Alias] self.alias_by_name = {} # type: typing.Dict[str, Alias] self.annotations = [] # type: typing.List[Annotation] self.annotation_by_name = {} # type: typing.Dict[str, Annotation] self.annotation_types = [] # type: typing.List[AnnotationType] self.annotation_type_by_name = {} # type: typing.Dict[str, AnnotationType] self._imported_namespaces = {} # type: typing.Dict[ApiNamespace, _ImportReason] def add_doc(self, docstring): # type: (typing.Any) -> None """Adds a docstring for this namespace. The input docstring is normalized to have no leading whitespace and no trailing whitespace except for a newline at the end. If a docstring already exists, the new normalized docstring is appended to the end of the existing one with two newlines separating them. """ assert isinstance(docstring, six.text_type), type(docstring) normalized_docstring = doc_unwrap(docstring) + '\n' if self.doc is None: self.doc = normalized_docstring else: self.doc += normalized_docstring def add_route(self, route): # type: (ApiRoute) -> None self.routes.append(route) if route.version == 1: self.route_by_name[route.name] = route if route.name not in self.routes_by_name: self.routes_by_name[route.name] = ApiRoutesByVersion() self.routes_by_name[route.name].at_version[route.version] = route def add_data_type(self, data_type): # type: (UserDefined) -> None self.data_types.append(data_type) self.data_type_by_name[data_type.name] = data_type def add_alias(self, alias): # type: (Alias) -> None self.aliases.append(alias) self.alias_by_name[alias.name] = alias def add_annotation(self, annotation): # type: (Annotation) -> None self.annotations.append(annotation) self.annotation_by_name[annotation.name] = annotation def add_annotation_type(self, annotation_type): # type: (AnnotationType) -> None self.annotation_types.append(annotation_type) self.annotation_type_by_name[annotation_type.name] = annotation_type def add_imported_namespace(self, namespace, imported_alias=False, imported_data_type=False, imported_annotation=False, imported_annotation_type=False): # type: (ApiNamespace, bool, bool, bool, bool) -> None """ Keeps track of namespaces that this namespace imports. Args: namespace (Namespace): The imported namespace. imported_alias (bool): Set if this namespace references an alias in the imported namespace. imported_data_type (bool): Set if this namespace references a data type in the imported namespace. imported_annotation (bool): Set if this namespace references a annotation in the imported namespace. imported_annotation_type (bool): Set if this namespace references an annotation in the imported namespace, possibly indirectly (by referencing an annotation elsewhere that has this type). """ assert self.name != namespace.name, \ 'Namespace cannot import itself.' reason = self._imported_namespaces.setdefault(namespace, _ImportReason()) if imported_alias: reason.alias = True if imported_data_type: reason.data_type = True if imported_annotation: reason.annotation = True if imported_annotation_type: reason.annotation_type = True def linearize_data_types(self): # type: () -> typing.List[UserDefined] """ Returns a list of all data types used in the namespace. Because the inheritance of data types can be modeled as a DAG, the list will be a linearization of the DAG. It's ideal to generate data types in this order so that composite types that reference other composite types are defined in the correct order. """ linearized_data_types = [] seen_data_types = set() # type: typing.Set[UserDefined] def add_data_type(data_type): # type: (UserDefined) -> None if data_type in seen_data_types: return elif data_type.namespace != self: # We're only concerned with types defined in this namespace. return if is_composite_type(data_type) and data_type.parent_type: add_data_type(data_type.parent_type) linearized_data_types.append(data_type) seen_data_types.add(data_type) for data_type in self.data_types: add_data_type(data_type) return linearized_data_types def linearize_aliases(self): # type: () -> typing.List[Alias] """ Returns a list of all aliases used in the namespace. The aliases are ordered to ensure that if they reference other aliases those aliases come earlier in the list. """ linearized_aliases = [] seen_aliases = set() # type: typing.Set[Alias] def add_alias(alias): # type: (Alias) -> None if alias in seen_aliases: return elif alias.namespace != self: return if is_alias(alias.data_type): add_alias(alias.data_type) linearized_aliases.append(alias) seen_aliases.add(alias) for alias in self.aliases: add_alias(alias) return linearized_aliases def get_route_io_data_types(self): # type: () -> typing.List[UserDefined] """ Returns a list of all user-defined data types that are referenced as either an argument, result, or error of a route. If a List or Nullable data type is referenced, then the contained data type is returned assuming it's a user-defined type. """ data_types = set() # type: typing.Set[UserDefined] for route in self.routes: data_types |= self.get_route_io_data_types_for_route(route) return sorted(data_types, key=lambda dt: dt.name) def get_route_io_data_types_for_route(self, route): # type: (ApiRoute) -> typing.Set[UserDefined] """ Given a route, returns a set of its argument/result/error datatypes. """ data_types = set() # type: typing.Set[UserDefined] for dtype in (route.arg_data_type, route.result_data_type, route.error_data_type): while is_list_type(dtype) or is_nullable_type(dtype): data_list_type = dtype # type: typing.Any dtype = data_list_type.data_type if is_composite_type(dtype) or is_alias(dtype): data_user_type = dtype # type: typing.Any data_types.add(data_user_type) return data_types def get_imported_namespaces(self, must_have_imported_data_type=False, consider_annotations=False, consider_annotation_types=False): # type: (bool, bool, bool) -> typing.List[ApiNamespace] """ Returns a list of Namespace objects. A namespace is a member of this list if it is imported by the current namespace and a data type is referenced from it. Namespaces are in ASCII order by name. Args: must_have_imported_data_type (bool): If true, result does not include namespaces that were not imported for data types. consider_annotations (bool): If false, result does not include namespaces that were only imported for annotations consider_annotation_types (bool): If false, result does not include namespaces that were only imported for annotation types. Returns: List[Namespace]: A list of imported namespaces. """ imported_namespaces = [] for imported_namespace, reason in self._imported_namespaces.items(): if must_have_imported_data_type and not reason.data_type: continue if (not consider_annotations) and not ( reason.data_type or reason.alias or reason.annotation_type ): continue if (not consider_annotation_types) and not ( reason.data_type or reason.alias or reason.annotation ): continue imported_namespaces.append(imported_namespace) imported_namespaces.sort(key=lambda n: n.name) return imported_namespaces def get_namespaces_imported_by_route_io(self): # type: () -> typing.List[ApiNamespace] """ Returns a list of Namespace objects. A namespace is a member of this list if it is imported by the current namespace and has a data type from it referenced as an argument, result, or error of a route. Namespaces are in ASCII order by name. """ namespace_data_types = sorted(self.get_route_io_data_types(), key=lambda dt: dt.name) referenced_namespaces = set() for data_type in namespace_data_types: if data_type.namespace != self: referenced_namespaces.add(data_type.namespace) return sorted(referenced_namespaces, key=lambda n: n.name) def normalize(self): # type: () -> None """ Alphabetizes routes to make route declaration order irrelevant. """ self.routes.sort(key=lambda route: route.name) self.data_types.sort(key=lambda data_type: data_type.name) self.aliases.sort(key=lambda alias: alias.name) self.annotations.sort(key=lambda annotation: annotation.name) def __repr__(self): # type: () -> str return str('ApiNamespace({!r})').format(self.name) class ApiRoute(object): """ Represents an API endpoint. """ def __init__(self, name, version, ast_node): # type: (typing.Text, int, typing.Optional[AstRouteDef]) -> None """ :param str name: Designated name of the endpoint. :param int version: Designated version of the endpoint. :param ast_node: Raw route definition from the parser. """ self.name = name self.version = version self._ast_node = ast_node # These attributes are set later by set_attributes() self.deprecated = None # type: typing.Optional[DeprecationInfo] self.raw_doc = None # type: typing.Optional[typing.Text] self.doc = None # type: typing.Optional[typing.Text] self.arg_data_type = None # type: typing.Optional[DataType] self.result_data_type = None # type: typing.Optional[DataType] self.error_data_type = None # type: typing.Optional[DataType] self.attrs = None # type: typing.Optional[typing.Mapping[typing.Text, typing.Any]] def set_attributes(self, deprecated, doc, arg_data_type, result_data_type, error_data_type, attrs): """ Converts a forward reference definition of a route into a full definition. :param DeprecationInfo deprecated: Set if this route is deprecated. :param str doc: Description of the endpoint. :type arg_data_type: :class:`stone.data_type.DataType` :type result_data_type: :class:`stone.data_type.DataType` :type error_data_type: :class:`stone.data_type.DataType` :param dict attrs: Map of string keys to values that are either int, float, bool, str, or None. These are the route attributes assigned in the spec. """ self.deprecated = deprecated self.raw_doc = doc self.doc = doc_unwrap(doc) self.arg_data_type = arg_data_type self.result_data_type = result_data_type self.error_data_type = error_data_type self.attrs = attrs def name_with_version(self): """ Get user-friendly representation of the route. :return: Route name with version suffix. The version suffix is omitted for version 1. """ if self.version == 1: return self.name else: return '{}:{}'.format(self.name, self.version) def __repr__(self): return 'ApiRoute({})'.format(self.name_with_version()) def _compare(self, lhs, rhs): if not isinstance(lhs, ApiRoute): raise TypeError("Expected ApiRoute for object: {}".format(lhs)) if not isinstance(rhs, ApiRoute): raise TypeError("Expected ApiRoute for object: {}".format(rhs)) if lhs.name < rhs.name or lhs.version < rhs.version: return -1 elif lhs.name > rhs.name or lhs.version > rhs.version: return 1 else: return 0 def __lt__(self, other): return self._compare(self, other) < 0 def __gt__(self, other): return self._compare(self, other) > 0 def __eq__(self, other): return self._compare(self, other) == 0 def __le__(self, other): return self._compare(self, other) <= 0 def __ge__(self, other): return self._compare(self, other) >= 0 def __ne__(self, other): return self._compare(self, other) != 0 def __hash__(self): return hash(( self.name, self.version, self._ast_node, self.deprecated, self.raw_doc, self.doc, self.arg_data_type, self.result_data_type, self.error_data_type, id(self.attrs) )) class DeprecationInfo(object): def __init__(self, by=None): # type: (typing.Optional[ApiRoute]) -> None """ :param ApiRoute by: The route that replaces this deprecated one. """ assert by is None or isinstance(by, ApiRoute), repr(by) self.by = by class ApiRoutesByVersion(object): """ Represents routes of different versions for a common name. """ def __init__(self): # type: () -> None """ :param at_version: The dict mapping a version number to a route. """ self.at_version = {} # type: typing.Dict[int, ApiRoute] stone-3.3.1/stone/ir/data_types.py000066400000000000000000002233661417406541500171420ustar00rootroot00000000000000""" Defines the Intermediate Representation that is generated by the frontend and fed to the backends. The goal of this module is to define all data types that are common to the languages and serialization formats we want to support. """ from __future__ import absolute_import, division, print_function, unicode_literals from abc import ABCMeta, abstractmethod from collections import OrderedDict, deque import copy import datetime import math import numbers import re import six from ..frontend.exception import InvalidSpec from ..frontend.ast import ( AstExampleField, AstExampleRef, AstTagRef, ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class ParameterError(Exception): """Raised when a data type is parameterized with a bad type or value.""" def generic_type_name(v): """ Return a descriptive type name that isn't Python specific. For example, an int type will return 'integer' rather than 'int'. """ if isinstance(v, AstExampleRef): return "reference" elif isinstance(v, numbers.Integral): # Must come before real numbers check since integrals are reals too return 'integer' elif isinstance(v, numbers.Real): return 'float' elif isinstance(v, (tuple, list)): return 'list' elif isinstance(v, six.string_types): return 'string' elif v is None: return 'null' else: return type(v).__name__ def record_custom_annotation_imports(annotation, namespace): """ Records imports for custom annotations in the given namespace. """ # first, check the annotation *type* if annotation.annotation_type.namespace.name != namespace.name: namespace.add_imported_namespace( annotation.annotation_type.namespace, imported_annotation_type=True) # second, check if we need to import the annotation itself # the annotation namespace is currently not actually used in the # backends, which reconstruct the annotation from the annotation # type directly. This could be changed in the future, and at # the IR level it makes sense to include the dependency if annotation.namespace.name != namespace.name: namespace.add_imported_namespace( annotation.namespace, imported_annotation=True) class DataType(object): """ Abstract class representing a data type. """ __metaclass__ = ABCMeta def __init__(self): """No-op. Exists so that introspection can be certain that an init method exists.""" @property def name(self): """Returns an easy to read name for the type.""" return self.__class__.__name__ @abstractmethod def check(self, val): """ Checks if a value specified in a spec (translated to a Python object) is a valid Python value for this type. Returns nothing, but can raise an exception. Args: val (object) Raises: ValueError """ @abstractmethod def check_example(self, ex_field): """ Checks if an example field from a spec is valid. Returns nothing, but can raise an exception. Args: ex_field (AstExampleField) Raises: InvalidSpec """ def __repr__(self): return self.name class Primitive(DataType): # pylint: disable=abstract-method def check_attr_repr(self, attr_field): try: self.check(attr_field.value) except ValueError as e: raise InvalidSpec(e.args[0], attr_field.lineno, attr_field.path) return attr_field.value class Composite(DataType): # pylint: disable=abstract-method """ Composite types are any data type which can be constructed using primitive data types and other composite types. """ def __init__(self): super(Composite, self).__init__() # contains custom annotations that apply to any containing data types (recursively) # format is (location, CustomAnnotation) to indicate a custom annotation is applied # to a location (Field or Alias) self.recursive_custom_annotations = None class Nullable(Composite): def __init__(self, data_type): super(Nullable, self).__init__() self.data_type = data_type def check(self, val): if val is not None: return self.data_type.check(val) def check_example(self, ex_field): if ex_field.value is not None: return self.data_type.check_example(ex_field) def check_attr_repr(self, attr_field): if attr_field.value is None: return None else: return self.data_type.check_attr_repr(attr_field) class Void(Primitive): def check(self, val): if val is not None: raise ValueError('void type can only be null') def check_example(self, ex_field): if ex_field.value is not None: raise InvalidSpec('example of void type must be null', ex_field.lineno, ex_field.path) def check_attr_repr(self, attr_field): raise NotImplementedError class Bytes(Primitive): def check(self, val): if not isinstance(val, (bytes, six.text_type)): raise ValueError('%r is not valid bytes' % val) def check_example(self, ex_field): if not isinstance(ex_field.value, (bytes, six.text_type)): raise InvalidSpec("'%s' is not valid bytes" % ex_field.value, ex_field.lineno, ex_field.path) def check_attr_repr(self, attr_field): try: self.check(attr_field.value) except ValueError as e: raise InvalidSpec(e.args[0], attr_field.lineno, attr_field.path) v = attr_field.value if isinstance(v, six.text_type): return v.encode('utf-8') else: return v class _BoundedInteger(Primitive): """ When extending, specify 'minimum' and 'maximum' as class variables. This is the range of values supported by the data type. """ # See minimum = None # type: typing.Optional[int] maximum = None # type: typing.Optional[int] def __init__(self, min_value=None, max_value=None): """ A more restrictive minimum or maximum value can be specified than the range inherent to the defined type. """ super(_BoundedInteger, self).__init__() if min_value is not None: if not isinstance(min_value, numbers.Integral): raise ParameterError('min_value must be an integral number') if min_value < self.minimum: raise ParameterError('min_value cannot be less than the ' 'minimum value for this type (%s < %s)' % (min_value, self.minimum)) if max_value is not None: if not isinstance(max_value, numbers.Integral): raise ParameterError('max_value must be an integral number') if max_value > self.maximum: raise ParameterError('max_value cannot be greater than the ' 'maximum value for this type (%s < %s)' % (max_value, self.maximum)) self.min_value = min_value self.max_value = max_value def check(self, val): if not isinstance(val, numbers.Integral): raise ValueError('%s is not a valid integer' % generic_type_name(val)) if not (self.minimum <= val <= self.maximum): raise ValueError('%d is not within range [%r, %r]' % (val, self.minimum, self.maximum)) if self.min_value is not None and val < self.min_value: raise ValueError('%d is less than %d' % (val, self.min_value)) if self.max_value is not None and val > self.max_value: raise ValueError('%d is greater than %d' % (val, self.max_value)) def check_example(self, ex_field): try: self.check(ex_field.value) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) def __repr__(self): return '%s()' % self.name class Int32(_BoundedInteger): minimum = -2**31 maximum = 2**31 - 1 class UInt32(_BoundedInteger): minimum = 0 maximum = 2**32 - 1 class Int64(_BoundedInteger): minimum = -2**63 maximum = 2**63 - 1 class UInt64(_BoundedInteger): minimum = 0 maximum = 2**64 - 1 class _BoundedFloat(Primitive): """ When extending, optionally specify 'minimum' and 'maximum' as class variables. This is the range of values supported by the data type. For a float64, there is no need to specify a minimum and maximum since Python's native float implementation is a float64/double. Therefore, any Python float will pass the data type range check automatically. """ # See minimum = None # type: typing.Optional[float] maximum = None # type: typing.Optional[float] def __init__(self, min_value=None, max_value=None): """ A more restrictive minimum or maximum value can be specified than the range inherent to the defined type. """ super(_BoundedFloat, self).__init__() if min_value is not None: if not isinstance(min_value, numbers.Real): raise ParameterError('min_value must be a real number') if not isinstance(min_value, float): try: min_value = float(min_value) except OverflowError: raise ParameterError('min_value is too small for a float') if self.minimum is not None and min_value < self.minimum: raise ParameterError( 'min_value cannot be less than the ' # pylint: disable=E1307 'minimum value for this type (%f < %f)' % (min_value, self.minimum) ) if max_value is not None: if not isinstance(max_value, numbers.Real): raise ParameterError('max_value must be a real number') if not isinstance(max_value, float): try: max_value = float(max_value) except OverflowError: raise ParameterError('max_value is too large for a float') if self.maximum is not None and max_value > self.maximum: raise ParameterError( 'max_value cannot be greater than the ' # pylint: disable=E1307 'maximum value for this type (%f < %f)' % (max_value, self.maximum) ) self.min_value = min_value self.max_value = max_value def check(self, val): if not isinstance(val, numbers.Real): raise ValueError('%s is not a valid real number' % generic_type_name(val)) if not isinstance(val, float): try: val = float(val) except OverflowError: raise ValueError('%r is too large for float' % val) if math.isnan(val) or math.isinf(val): # Parser doesn't support NaN or Inf yet. raise ValueError('%f values are not supported' % val) if self.minimum is not None and val < self.minimum: raise ValueError( '%f is less than %f' % # pylint: disable=E1307 (val, self.minimum) ) if self.maximum is not None and val > self.maximum: raise ValueError( '%f is greater than %f' % # pylint: disable=E1307 (val, self.maximum) ) if self.min_value is not None and val < self.min_value: raise ValueError('%f is less than %f' % (val, self.min_value)) if self.max_value is not None and val > self.max_value: raise ValueError('%f is greater than %f' % (val, self.min_value)) def check_example(self, ex_field): try: self.check(ex_field.value) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) def __repr__(self): return '%s()' % self.name class Float32(_BoundedFloat): # Maximum and minimums from the IEEE 754-1985 standard minimum = -3.40282 * 10**38 maximum = 3.40282 * 10**38 class Float64(_BoundedFloat): pass class Boolean(Primitive): def check(self, val): if not isinstance(val, bool): raise ValueError('%r is not a valid boolean' % val) def check_example(self, ex_field): try: self.check(ex_field.value) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) class String(Primitive): def __init__(self, min_length=None, max_length=None, pattern=None): super(String, self).__init__() if min_length is not None: if not isinstance(min_length, numbers.Integral): raise ParameterError('min_length must be an integral number') if min_length < 0: raise ParameterError('min_length must be >= 0') if max_length is not None: if not isinstance(max_length, numbers.Integral): raise ParameterError('max_length must be an integral number') if max_length < 1: raise ParameterError('max_length must be > 0') if min_length and max_length: if max_length < min_length: raise ParameterError('max_length must be >= min_length') self.min_length = min_length self.max_length = max_length self.pattern = pattern self.pattern_re = None if pattern: if not isinstance(pattern, six.string_types): raise ParameterError('pattern must be a string') try: self.pattern_re = re.compile(pattern) except re.error as e: raise ParameterError( 'could not compile regex pattern {!r}: {}'.format( pattern, e.args[0])) def check(self, val): if not isinstance(val, six.string_types): raise ValueError('%s is not a valid string' % generic_type_name(val)) elif self.max_length is not None and len(val) > self.max_length: raise ValueError("'%s' has more than %d character(s)" % (val, self.max_length)) elif self.min_length is not None and len(val) < self.min_length: raise ValueError("'%s' has fewer than %d character(s)" % (val, self.min_length)) elif self.pattern and not self.pattern_re.match(val): raise ValueError("'%s' did not match pattern '%s'" % (val, self.pattern)) def check_example(self, ex_field): try: self.check(ex_field.value) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) class Timestamp(Primitive): def __init__(self, fmt): super(Timestamp, self).__init__() if not isinstance(fmt, six.string_types): raise ParameterError('format must be a string') self.format = fmt def check(self, val): if not isinstance(val, six.string_types): raise ValueError('timestamp must be specified as a string') # Raises a ValueError if val is the incorrect format datetime.datetime.strptime(val, self.format) def check_example(self, ex_field): try: self.check(ex_field.value) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) def check_attr_repr(self, attr_field): try: self.check(attr_field.value) except ValueError as e: msg = e.args[0] if isinstance(msg, six.binary_type): # For Python 2 compatibility. msg = msg.decode('utf-8') raise InvalidSpec(msg, attr_field.lineno, attr_field.path) return datetime.datetime.strptime(attr_field.value, self.format) class List(Composite): def __init__(self, data_type, min_items=None, max_items=None): super(List, self).__init__() self.data_type = data_type if min_items is not None and min_items < 0: raise ParameterError('min_items must be >= 0') if max_items is not None and max_items < 1: raise ParameterError('max_items must be > 0') if min_items and max_items and max_items < min_items: raise ParameterError('max_length must be >= min_length') self.min_items = min_items self.max_items = max_items def check(self, val): raise NotImplementedError def check_example(self, ex_field): try: self._check_list_container(ex_field.value) for item in ex_field.value: new_ex_field = AstExampleField( ex_field.path, ex_field.lineno, ex_field.lexpos, ex_field.name, item) self.data_type.check_example(new_ex_field) except ValueError as e: raise InvalidSpec(e.args[0], ex_field.lineno, ex_field.path) def _check_list_container(self, val): if not isinstance(val, list): raise ValueError('%s is not a valid list' % generic_type_name(val)) elif self.max_items is not None and len(val) > self.max_items: raise ValueError('list has more than %s item(s)' % self.max_items) elif self.min_items is not None and len(val) < self.min_items: raise ValueError('list has fewer than %s item(s)' % self.min_items) class Map(Composite): def __init__(self, key_data_type, value_data_type): super(Map, self).__init__() if not isinstance(key_data_type, String): raise ParameterError("Only String primitives are supported as key types.") self.key_data_type = key_data_type self.value_data_type = value_data_type def check(self, val): raise NotImplementedError def check_example(self, ex_field): if not isinstance(ex_field.value, dict): raise ValueError("%s is not a valid map" % generic_type_name(ex_field.value)) for k, v in ex_field.value.items(): ex_key_field = self._make_ex_field(ex_field, k) ex_value_field = self._make_ex_field(ex_field, v) self.key_data_type.check_example(ex_key_field) self.value_data_type.check_example(ex_value_field) def _make_ex_field(self, ex_field, value): return AstExampleField( ex_field.path, ex_field.lineno, ex_field.lexpos, ex_field.name, value) def doc_unwrap(raw_doc): """ Applies two transformations to raw_doc: 1. N consecutive newlines are converted into N-1 newlines. 2. A lone newline is converted to a space, which basically unwraps text. Returns a new string, or None if the input was None. """ if raw_doc is None: return None docstring = '' consecutive_newlines = 0 # Remove all leading and trailing whitespace in the documentation block for c in raw_doc.strip(): if c == '\n': consecutive_newlines += 1 if consecutive_newlines > 1: docstring += c else: if consecutive_newlines == 1: docstring += ' ' consecutive_newlines = 0 docstring += c return docstring class Field(object): """ Represents a field in a composite type. """ def __init__(self, name, data_type, doc, ast_node): """ Creates a new Field. :param str name: Name of the field. :param Type data_type: The type of variable for of this field. :param str doc: Documentation for the field. :param ast_node: Raw field definition from the parser. :type ast_node: stone.frontend.ast.AstField """ self.name = name self.data_type = data_type self.raw_doc = doc self.doc = doc_unwrap(doc) self._ast_node = ast_node self.redactor = None self.omitted_caller = None self.deprecated = None self.preview = None self.custom_annotations = [] def set_annotations(self, annotations): if not annotations: return for annotation in annotations: if isinstance(annotation, Deprecated): if self.deprecated: raise InvalidSpec("Deprecated value already set as %r." % str(self.deprecated), self._ast_node.lineno) if self.preview: raise InvalidSpec("'Deprecated' and 'Preview' can\'t both be set.", self._ast_node.lineno) self.deprecated = True self.doc = 'Field is deprecated. {}'.format(self.doc) elif isinstance(annotation, Omitted): if self.omitted_caller: raise InvalidSpec("Omitted caller already set as %r." % str(self.omitted_caller), self._ast_node.lineno) self.omitted_caller = annotation.omitted_caller self.doc = 'Field is only returned for "{}" callers. {}'.format( str(self.omitted_caller), self.doc) elif isinstance(annotation, Preview): if self.preview: raise InvalidSpec("Preview value already set as %r." % str(self.preview), self._ast_node.lineno) if self.deprecated: raise InvalidSpec("'Deprecated' and 'Preview' can\'t both be set.", self._ast_node.lineno) self.preview = True self.doc = 'Field is in preview mode - do not rely on in production. {}'.format( self.doc ) elif isinstance(annotation, Redacted): # Make sure we don't set multiple conflicting annotations on one field if self.redactor: raise InvalidSpec("Redactor already set as %r." % str(self.redactor), self._ast_node.lineno) self.redactor = annotation elif isinstance(annotation, CustomAnnotation): self.custom_annotations.append(annotation) else: raise InvalidSpec( 'Annotation %r not recognized for field.' % annotation, self._ast_node.lineno) def __repr__(self): return 'Field(%r, %r)' % (self.name, self.data_type) class StructField(Field): """ Represents a field of a struct. """ def __init__(self, name, data_type, doc, ast_node): """ Creates a new Field. :param str name: Name of the field. :param Type data_type: The type of variable for of this field. :param str doc: Documentation for the field. :param ast_node: Raw field definition from the parser. :type ast_node: stone.frontend.ast.AstField """ super(StructField, self).__init__(name, data_type, doc, ast_node) self.has_default = False self._default = None def set_default(self, default): self.has_default = True self._default = default @property def default(self): if not self.has_default: raise Exception('Type has no default') else: return self._default def check_attr_repr(self, attr): if attr is not None: attr = self.data_type.check_attr_repr(attr) if attr is None: if self.has_default: return self.default _, unwrapped_nullable, _ = unwrap(self.data_type) if unwrapped_nullable: return None else: raise KeyError(self.name) return attr def __repr__(self): return 'StructField(%r, %r, %r)' % (self.name, self.data_type, self.omitted_caller) class UnionField(Field): """ Represents a field of a union. """ def __init__(self, name, data_type, doc, ast_node, catch_all=False): super(UnionField, self).__init__(name, data_type, doc, ast_node) self.catch_all = catch_all def __repr__(self): return 'UnionField(%r, %r, %r, %r)' % (self.name, self.data_type, self.catch_all, self.omitted_caller) class UserDefined(Composite): """ These are types that are defined directly in specs. """ DEFAULT_EXAMPLE_LABEL = 'default' def __init__(self, name, namespace, ast_node): """ When this is instantiated, the type is treated as a forward reference. Only when :meth:`set_attributes` is called is the type considered to be fully defined. :param str name: Name of type. :param stone.ir.Namespace namespace: The namespace this type is defined in. :param ast_node: Raw type definition from the parser. :type ast_node: stone.frontend.ast.AstTypeDef """ super(UserDefined, self).__init__() self._name = name self.namespace = namespace self._ast_node = ast_node self._is_forward_ref = True self.raw_doc = None self.doc = None self.fields = None self.parent_type = None self._raw_examples = None self._examples = None self._fields_by_name = None def set_attributes(self, doc, fields, parent_type=None): """ Fields are specified as a list so that order is preserved for display purposes only. (Might be used for certain serialization formats...) :param str doc: Description of type. :param list(Field) fields: Ordered list of fields for type. :param Optional[Composite] parent_type: The type this type inherits from. """ self.raw_doc = doc self.doc = doc_unwrap(doc) self.fields = fields self.parent_type = parent_type self._raw_examples = OrderedDict() self._examples = OrderedDict() self._fields_by_name = {} # Dict[str, Field] # Check that no two fields share the same name. for field in self.fields: if field.name in self._fields_by_name: orig_lineno = self._fields_by_name[field.name]._ast_node.lineno raise InvalidSpec("Field '%s' already defined on line %s." % (field.name, orig_lineno), field._ast_node.lineno) self._fields_by_name[field.name] = field # Check that the fields for this type do not match any of the fields of # its parents. cur_type = self.parent_type while cur_type: for field in self.fields: if field.name in cur_type._fields_by_name: lineno = cur_type._fields_by_name[field.name]._ast_node.lineno raise InvalidSpec( "Field '%s' already defined in parent '%s' on line %d." % (field.name, cur_type.name, lineno), field._ast_node.lineno) cur_type = cur_type.parent_type # Import namespaces containing any custom annotations # Note: we don't need to do this for builtin annotations because # they are treated as globals at the IR level for field in self.fields: for annotation in field.custom_annotations: record_custom_annotation_imports(annotation, self.namespace) # Indicate that the attributes of the type have been populated. self._is_forward_ref = False @property def all_fields(self): raise NotImplementedError def has_documented_type_or_fields(self, include_inherited_fields=False): """Returns whether this type, or any of its fields, are documented. Use this when deciding whether to create a block of documentation for this type. """ if self.doc: return True else: return self.has_documented_fields(include_inherited_fields) def has_documented_fields(self, include_inherited_fields=False): """Returns whether at least one field is documented.""" fields = self.all_fields if include_inherited_fields else self.fields for field in fields: if field.doc: return True return False def get_all_omitted_callers(self): """Returns all unique omitted callers for the object.""" return {f.omitted_caller for f in self.fields if f.omitted_caller} @property def name(self): return self._name def copy(self): return copy.deepcopy(self) def prepend_field(self, field): self.fields.insert(0, field) def get_examples(self, compact=False): """ Returns an OrderedDict mapping labels to Example objects. Args: compact (bool): If True, union members of void type are converted to their compact representation: no ".tag" key or containing dict, just the tag as a string. """ # Copy it just in case the caller wants to mutate the object. examples = copy.deepcopy(self._examples) if not compact: return examples def make_compact(d): # Traverse through dicts looking for ones that have a lone .tag # key, which can be converted into the compact form. if not isinstance(d, dict): return for key in d: if isinstance(d[key], dict): inner_d = d[key] if len(inner_d) == 1 and '.tag' in inner_d: d[key] = inner_d['.tag'] else: make_compact(inner_d) if isinstance(d[key], list): for item in d[key]: make_compact(item) for example in examples.values(): if (isinstance(example.value, dict) and len(example.value) == 1 and '.tag' in example.value): # Handle the case where the top-level of the example can be # made compact. example.value = example.value['.tag'] else: make_compact(example.value) return examples class Example(object): """An example of a struct or union type.""" def __init__(self, label, text, value, ast_node=None): assert isinstance(label, six.text_type), type(label) self.label = label assert isinstance(text, (six.text_type, type(None))), type(text) self.text = doc_unwrap(text) if text else text assert isinstance(value, (six.text_type, OrderedDict)), type(value) self.value = value self._ast_node = ast_node def __repr__(self): return 'Example({!r}, {!r}, {!r})'.format( self.label, self.text, self.value) class Struct(UserDefined): """ Defines a product type: Composed of other primitive and/or struct types. """ composite_type = 'struct' def set_attributes(self, doc, fields, parent_type=None): """ See :meth:`Composite.set_attributes` for parameter definitions. """ if parent_type: assert isinstance(parent_type, Struct) self.subtypes = [] # These are only set if this struct enumerates subtypes. self._enumerated_subtypes = None # Optional[List[Tuple[str, DataType]]] self._is_catch_all = None # Optional[Bool] super(Struct, self).set_attributes(doc, fields, parent_type) if self.parent_type: self.parent_type.subtypes.append(self) def check(self, val): raise NotImplementedError def check_example(self, ex_field): if not isinstance(ex_field.value, AstExampleRef): raise InvalidSpec( "example must reference label of '%s'" % self.name, ex_field.lineno, ex_field.path) def check_attr_repr(self, attrs): # Since we mutate it, let's make a copy to avoid mutating the argument. attrs = attrs.copy() validated_attrs = {} for field in self.all_fields: attr = field.check_attr_repr(attrs.pop(field.name, None)) validated_attrs[field.name] = attr if attrs: attr_name, attr_field = attrs.popitem() raise InvalidSpec( "Route attribute '%s' is not defined in 'stone_cfg.Route'." % attr_name, attr_field.lineno, attr_field.path) return validated_attrs @property def all_fields(self): """ Returns an iterator of all fields. Required fields before optional fields. Super type fields before type fields. """ return self.all_required_fields + self.all_optional_fields def _filter_fields(self, filter_function): """ Utility to iterate through all fields (super types first) of a type. :param filter: A function that takes in a Field object. If it returns True, the field is part of the generated output. If False, it is omitted. """ fields = [] if self.parent_type: fields.extend(self.parent_type._filter_fields(filter_function)) fields.extend(filter(filter_function, self.fields)) return fields @property def all_required_fields(self): """ Returns an iterator that traverses required fields in all super types first, and then for this type. """ def required_check(f): return not is_nullable_type(f.data_type) and not f.has_default return self._filter_fields(required_check) @property def all_optional_fields(self): """ Returns an iterator that traverses optional fields in all super types first, and then for this type. """ def optional_check(f): return is_nullable_type(f.data_type) or f.has_default return self._filter_fields(optional_check) def has_enumerated_subtypes(self): """ Whether this struct enumerates its subtypes. """ return bool(self._enumerated_subtypes) def get_enumerated_subtypes(self): """ Returns a list of subtype fields. Each field has a `name` attribute which is the tag for the subtype. Each field also has a `data_type` attribute that is a `Struct` object representing the subtype. """ assert self._enumerated_subtypes is not None return self._enumerated_subtypes def is_member_of_enumerated_subtypes_tree(self): """ Whether this struct enumerates subtypes or is a struct that is enumerated by its parent type. Because such structs are serialized and deserialized differently, use this method to detect these. """ return (self.has_enumerated_subtypes() or (self.parent_type and self.parent_type.has_enumerated_subtypes())) def is_catch_all(self): """ Indicates whether this struct should be used in the event that none of its known enumerated subtypes match a received type tag. Use this method only if the struct has enumerated subtypes. Returns: bool """ assert self._enumerated_subtypes is not None return self._is_catch_all def set_enumerated_subtypes(self, subtype_fields, is_catch_all): """ Sets the list of "enumerated subtypes" for this struct. This differs from regular subtyping in that each subtype is associated with a tag that is used in the serialized format to indicate the subtype. Also, this list of subtypes was explicitly defined in an "inner-union" in the specification. The list of fields must include all defined subtypes of this struct. NOTE(kelkabany): For this to work with upcoming forward references, the hierarchy of parent types for this struct must have had this method called on them already. :type subtype_fields: List[UnionField] """ assert self._enumerated_subtypes is None, \ 'Enumerated subtypes already set.' assert isinstance(is_catch_all, bool), type(is_catch_all) self._is_catch_all = is_catch_all self._enumerated_subtypes = [] if self.parent_type: raise InvalidSpec( "'%s' enumerates subtypes so it cannot extend another struct." % self.name, self._ast_node.lineno, self._ast_node.path) # Require that if this struct enumerates subtypes, its parent (and thus # the entire hierarchy above this struct) does as well. if self.parent_type and not self.parent_type.has_enumerated_subtypes(): raise InvalidSpec( "'%s' cannot enumerate subtypes if parent '%s' does not." % (self.name, self.parent_type.name), self._ast_node.lineno, self._ast_node.path) enumerated_subtype_names = set() # Set[str] for subtype_field in subtype_fields: path = subtype_field._ast_node.path lineno = subtype_field._ast_node.lineno # Require that a subtype only has a single type tag. if subtype_field.data_type.name in enumerated_subtype_names: raise InvalidSpec( "Subtype '%s' can only be specified once." % subtype_field.data_type.name, lineno, path) # Require that a subtype has this struct as its parent. if subtype_field.data_type.parent_type != self: raise InvalidSpec( "'%s' is not a subtype of '%s'." % (subtype_field.data_type.name, self.name), lineno, path) # Check for subtype tags that conflict with this struct's # non-inherited fields. if subtype_field.name in self._fields_by_name: # Since the union definition comes first, use its line number # as the source of the field's original declaration. orig_field = self._fields_by_name[subtype_field.name] raise InvalidSpec( "Field '%s' already defined on line %d." % (subtype_field.name, lineno), orig_field._ast_node.lineno, orig_field._ast_node.path) # Walk up parent tree hierarchy to ensure no field conflicts. # Checks for conflicts with subtype tags and regular fields. cur_type = self.parent_type while cur_type: if subtype_field.name in cur_type._fields_by_name: orig_field = cur_type._fields_by_name[subtype_field.name] raise InvalidSpec( "Field '%s' already defined in parent '%s' (%s:%d)." % (subtype_field.name, cur_type.name, orig_field._ast_node.path, orig_field._ast_node.lineno), lineno, path) cur_type = cur_type.parent_type # Note the discrepancy between `fields` which contains only the # struct fields, and `_fields_by_name` which contains the struct # fields and enumerated subtype fields. self._fields_by_name[subtype_field.name] = subtype_field enumerated_subtype_names.add(subtype_field.data_type.name) self._enumerated_subtypes.append(subtype_field) assert len(self._enumerated_subtypes) > 0 # Check that all known subtypes are listed in the enumeration. for subtype in self.subtypes: if subtype.name not in enumerated_subtype_names: raise InvalidSpec( "'%s' does not enumerate all subtypes, missing '%s'" % (self.name, subtype.name), self._ast_node.lineno) def get_all_subtypes_with_tags(self): """ Unlike other enumerated-subtypes-related functionality, this method returns not just direct subtypes, but all subtypes of this struct. The tag of each subtype is the list of tags from which the type descends. This method only applies to structs that enumerate subtypes. Use this when you need to generate a lookup table for a root struct that maps a generated class representing a subtype to the tag it needs in the serialized format. Returns: List[Tuple[List[String], Struct]] """ assert self.has_enumerated_subtypes(), 'Enumerated subtypes not set.' subtypes_with_tags = [] # List[Tuple[List[String], Struct]] fifo = deque([subtype_field.data_type for subtype_field in self.get_enumerated_subtypes()]) # Traverse down the hierarchy registering subtypes as they're found. while fifo: data_type = fifo.popleft() subtypes_with_tags.append((data_type._get_subtype_tags(), data_type)) if data_type.has_enumerated_subtypes(): for subtype_field in data_type.get_enumerated_subtypes(): fifo.append(subtype_field.data_type) return subtypes_with_tags def _get_subtype_tags(self): """ Returns a list of type tags that refer to this type starting from the base of the struct hierarchy. """ assert self.is_member_of_enumerated_subtypes_tree(), \ 'Not a part of a subtypes tree.' cur = self.parent_type cur_dt = self tags = [] while cur: assert cur.has_enumerated_subtypes() for subtype_field in cur.get_enumerated_subtypes(): if subtype_field.data_type is cur_dt: tags.append(subtype_field.name) break else: assert False, 'Could not find?!' cur_dt = cur cur = cur.parent_type tags.reverse() return tuple(tags) def _add_example(self, example): """Adds a "raw example" for this type. This does basic sanity checking to ensure that the example is valid (required fields specified, no unknown fields, correct types, ...). The example is not available via :meth:`get_examples` until :meth:`_compute_examples` is called. Args: example (stone.frontend.ast.AstExample): An example of this type. """ if self.has_enumerated_subtypes(): self._add_example_enumerated_subtypes_helper(example) else: self._add_example_helper(example) def _add_example_enumerated_subtypes_helper(self, example): """Validates examples for structs with enumerated subtypes.""" if len(example.fields) != 1: raise InvalidSpec( 'Example for struct with enumerated subtypes must only ' 'specify one subtype tag.', example.lineno, example.path) # Extract the only tag in the example. example_field = list(example.fields.values())[0] tag = example_field.name val = example_field.value if not isinstance(val, AstExampleRef): raise InvalidSpec( "Example of struct with enumerated subtypes must be a " "reference to a subtype's example.", example_field.lineno, example_field.path) for subtype_field in self.get_enumerated_subtypes(): if subtype_field.name == tag: self._raw_examples[example.label] = example break else: raise InvalidSpec( "Unknown subtype tag '%s' in example." % tag, example_field.lineno, example_field.path) def _add_example_helper(self, example): """Validates examples for structs without enumerated subtypes.""" # Check for fields in the example that don't belong. for label, example_field in example.fields.items(): if not any(label == f.name for f in self.all_fields): raise InvalidSpec( "Example for '%s' has unknown field '%s'." % (self.name, label), example_field.lineno, example_field.path, ) for field in self.all_fields: if field.name in example.fields: example_field = example.fields[field.name] try: field.data_type.check_example(example_field) except InvalidSpec as e: e.msg = "Bad example for field '{}': {}".format( field.name, e.msg) raise elif field.has_default or isinstance(field.data_type, Nullable): # These don't need examples. pass else: raise InvalidSpec( "Missing field '%s' in example." % field.name, example.lineno, example.path) self._raw_examples[example.label] = example def _has_example(self, label): """Whether this data type has an example with the given ``label``.""" return label in self._raw_examples def _compute_examples(self): """ Populates the ``_examples`` instance attribute by computing full examples for each label in ``_raw_examples``. The logic in this method is separate from :meth:`_add_example` because this method requires that every type have ``_raw_examples`` assigned for resolving example references. """ for label in self._raw_examples: self._examples[label] = self._compute_example(label) def _compute_example(self, label): if self.has_enumerated_subtypes(): return self._compute_example_enumerated_subtypes(label) else: return self._compute_example_flat_helper(label) def _compute_example_flat_helper(self, label): """ From the "raw example," resolves references to examples of other data types to compute the final example. Returns an Example object. The `value` attribute contains a JSON-serializable representation of the example. """ assert label in self._raw_examples, label example = self._raw_examples[label] def deref_example_ref(dt, val): dt, _ = unwrap_nullable(dt) if not dt._has_example(val.label): raise InvalidSpec( "Reference to example for '%s' with label '%s' " "does not exist." % (dt.name, val.label), val.lineno, val.path) return dt._compute_example(val.label).value # Do a deep copy of the example because we're going to mutate it. ex_val = OrderedDict() def get_json_val(dt, val): if isinstance(val, AstExampleRef): # Embed references to other examples directly. return deref_example_ref(dt, val) elif isinstance(val, TagRef): return val.union_data_type._compute_example(val.tag_name).value elif isinstance(val, list): dt, _ = unwrap_nullable(dt) return [get_json_val(dt.data_type, v) for v in val] elif isinstance(val, dict): dt, _ = unwrap_nullable(dt) if is_alias(dt): return val return {k: get_json_val(dt.value_data_type, v) for (k, v) in val.items()} else: return val for field in self.all_fields: if field.name in example.fields: example_field = example.fields[field.name] if example_field.value is None: # Serialized format doesn't include fields with null. pass else: ex_val[field.name] = get_json_val( field.data_type, example_field.value) elif field.has_default: ex_val[field.name] = get_json_val( field.data_type, field.default) return Example(example.label, example.text, ex_val, ast_node=example) def _compute_example_enumerated_subtypes(self, label): """ Analogous to :meth:`_compute_example_flat_helper` but for structs with enumerated subtypes. """ assert label in self._raw_examples, label example = self._raw_examples[label] example_field = list(example.fields.values())[0] for subtype_field in self.get_enumerated_subtypes(): if subtype_field.name == example_field.name: data_type = subtype_field.data_type break ref = example_field.value if not data_type._has_example(ref.label): raise InvalidSpec( "Reference to example for '%s' with label '%s' does not " "exist." % (data_type.name, ref.label), ref.lineno, ref.path) ordered_value = OrderedDict([('.tag', example_field.name)]) flat_example = data_type._compute_example_flat_helper(ref.label) ordered_value.update(flat_example.value) flat_example.value = ordered_value return flat_example def __repr__(self): return 'Struct(%r, %r)' % (self.name, self.fields) class Union(UserDefined): """Defines a tagged union. Fields are variants.""" composite_type = 'union' def __init__(self, name, namespace, ast_node, closed): super(Union, self).__init__(name, namespace, ast_node) self.closed = closed # TODO: Why is this a different signature than the parent? Is this # intentional? def set_attributes(self, doc, fields, # pylint: disable=arguments-differ parent_type=None, catch_all_field=None): """ :param UnionField catch_all_field: The field designated as the catch-all. This field should be a member of the list of fields. See :meth:`Composite.set_attributes` for parameter definitions. """ if parent_type: assert isinstance(parent_type, Union) super(Union, self).set_attributes(doc, fields, parent_type) self.catch_all_field = catch_all_field self.parent_type = parent_type def check(self, val): assert isinstance(val, TagRef) for field in self.all_fields: if val.tag_name == field.name: if not is_void_type(field.data_type): raise ValueError( "invalid reference to non-void option '%s'" % val.tag_name) break else: raise ValueError( "invalid reference to unknown tag '%s'" % val.tag_name) def check_example(self, ex_field): if not isinstance(ex_field.value, AstExampleRef): raise InvalidSpec( "example must reference label of '%s'" % self.name, ex_field.lineno, ex_field.path) def check_attr_repr(self, attr_field): if not isinstance(attr_field.value, AstTagRef): raise InvalidSpec( 'Expected union tag as value.', attr_field.lineno, attr_field.path) tag_ref = TagRef(self, attr_field.value.tag) try: self.check(tag_ref) except ValueError as e: raise InvalidSpec(e.args[0], attr_field.lineno, attr_field.path) return tag_ref @property def all_fields(self): """ Returns a list of all fields. Subtype fields come before this type's fields. """ fields = [] if self.parent_type: fields.extend(self.parent_type.all_fields) fields.extend([f for f in self.fields]) return fields def _add_example(self, example): """Adds a "raw example" for this type. This does basic sanity checking to ensure that the example is valid (required fields specified, no unknown fields, correct types, ...). The example is not available via :meth:`get_examples` until :meth:`_compute_examples` is called. Args: example (stone.frontend.ast.AstExample): An example of this type. """ if len(example.fields) != 1: raise InvalidSpec( 'Example for union must specify exactly one tag.', example.lineno, example.path) # Extract the only tag in the example. example_field = list(example.fields.values())[0] tag = example_field.name # Find the union member that corresponds to the tag. for field in self.all_fields: if tag == field.name: break else: # Error: Tag doesn't match any union member. raise InvalidSpec( "Unknown tag '%s' in example." % tag, example.lineno, example.path ) # TODO: are we always guaranteed at least one field? # pylint: disable=undefined-loop-variable try: field.data_type.check_example(example_field) except InvalidSpec as e: e.msg = "Bad example for field '{}': {}".format( field.name, e.msg) raise self._raw_examples[example.label] = example def _has_example(self, label): """Whether this data type has an example with the given ``label``.""" if label in self._raw_examples: return True else: for field in self.all_fields: dt, _ = unwrap_nullable(field.data_type) if not is_user_defined_type(dt) and not is_void_type(dt): continue if label == field.name: return True else: return False def _compute_examples(self): """ Populates the ``_examples`` instance attribute by computing full examples for each label in ``_raw_examples``. The logic in this method is separate from :meth:`_add_example` because this method requires that every type have ``_raw_examples`` assigned for resolving example references. """ for label in self._raw_examples: self._examples[label] = self._compute_example(label) # Add examples for each void union member. for field in self.all_fields: dt, _ = unwrap_nullable(field.data_type) if is_void_type(dt): self._examples[field.name] = \ Example( field.name, None, OrderedDict([('.tag', field.name)])) def _compute_example(self, label): """ From the "raw example," resolves references to examples of other data types to compute the final example. Returns an Example object. The `value` attribute contains a JSON-serializable representation of the example. """ if label in self._raw_examples: example = self._raw_examples[label] def deref_example_ref(dt, val): dt, _ = unwrap_nullable(dt) if not dt._has_example(val.label): raise InvalidSpec( "Reference to example for '%s' with label '%s' " "does not exist." % (dt.name, val.label), val.lineno, val.path) return dt._compute_example(val.label).value def get_json_val(dt, val): if isinstance(val, AstExampleRef): # Embed references to other examples directly. return deref_example_ref(dt, val) elif isinstance(val, list): return [get_json_val(dt.data_type, v) for v in val] else: return val example_field = list(example.fields.values())[0] # Do a deep copy of the example because we're going to mutate it. ex_val = OrderedDict([('.tag', example_field.name)]) for field in self.all_fields: if field.name == example_field.name: break # TODO: are we always guaranteed at least one field? # pylint: disable=undefined-loop-variable data_type, _ = unwrap_nullable(field.data_type) inner_ex_val = get_json_val(data_type, example_field.value) if (isinstance(data_type, Struct) and not data_type.has_enumerated_subtypes()): ex_val.update(inner_ex_val) else: if inner_ex_val is not None: ex_val[field.name] = inner_ex_val return Example(example.label, example.text, ex_val, ast_node=example) else: # Try to fallback to a union member with tag matching the label # with a data type that is composite or void. for field in self.all_fields: if label == field.name: break else: raise AssertionError('No example for label %r' % label) # TODO: are we always guaranteed at least one field? # pylint: disable=undefined-loop-variable assert is_void_type(field.data_type) return Example( field.name, field.doc, OrderedDict([('.tag', field.name)])) def unique_field_data_types(self): """ Checks if all variants have different data types. If so, the selected variant can be determined just by the data type of the value without needing a field name / tag. In some languages, this lets us make a shortcut """ data_type_names = set() for field in self.fields: if not is_void_type(field.data_type): if field.data_type.name in data_type_names: return False else: data_type_names.add(field.data_type.name) else: return True def __repr__(self): return 'Union(%r, %r)' % (self.name, self.fields) class TagRef(object): """ Used when an ID in Stone refers to a tag of a union. TODO(kelkabany): Support tag values. """ def __init__(self, union_data_type, tag_name): self.union_data_type = union_data_type self.tag_name = tag_name def __repr__(self): return 'TagRef(%r, %r)' % (self.union_data_type, self.tag_name) class AnnotationTypeParam(object): """ A parameter that can be supplied to a custom annotation type. """ def __init__(self, name, data_type, doc, has_default, default, ast_node): self.name = name self.data_type = data_type self.raw_doc = doc self.doc = doc_unwrap(doc) self.has_default = has_default self.default = default self._ast_node = ast_node if self.has_default: try: self.data_type.check(self.default) except ValueError as e: raise InvalidSpec('Default value for parameter %s is invalid: %s' % ( self.name, e), self._ast_node.lineno, self._ast_node.path) class AnnotationType(object): """ Used when a spec defines a custom annotation type. """ def __init__(self, name, namespace, doc, params): self.name = name self.namespace = namespace self.raw_doc = doc self.doc = doc_unwrap(doc) self.params = params self._params_by_name = {} # type: typing.Dict[str, AnnotationTypeParam] for param in self.params: if param.name in self._params_by_name: orig_lineno = self._params_by_name[param.name]._ast_node.lineno raise InvalidSpec("Parameter '%s' already defined on line %s." % (param.name, orig_lineno), param._ast_node.lineno, param._ast_node.path) self._params_by_name[param.name] = param def has_documented_type_or_params(self): """Returns whether this type, or any of its parameters, are documented. Use this when deciding whether to create a block of documentation for this type. """ return self.doc or self.has_documented_params() def has_documented_params(self): """Returns whether at least one param is documented.""" return any(param.doc for param in self.params) class Annotation(object): """ Used when a field is annotated with a pre-defined Stone action or a custom annotation. """ def __init__(self, name, namespace, ast_node): self.name = name self.namespace = namespace self._ast_node = ast_node class Deprecated(Annotation): """ Used when a field is annotated for deprecation. """ def __repr__(self): return 'Deprecated(%r, %r)' % (self.name, self.namespace) class Omitted(Annotation): """ Used when a field is annotated for omission. """ def __init__(self, name, namespace, ast_node, omitted_caller): super(Omitted, self).__init__(name, namespace, ast_node) self.omitted_caller = omitted_caller def __repr__(self): return 'Omitted(%r, %r, %r)' % (self.name, self.namespace, self.omitted_caller) class Preview(Annotation): """ Used when a field is annotated for previewing. """ def __repr__(self): return 'Preview(%r, %r)' % (self.name, self.namespace) class Redacted(Annotation): """ Used when a field is annotated for redaction. """ def __init__(self, name, namespace, ast_node, regex=None): super(Redacted, self).__init__(name, namespace, ast_node) self.regex = regex class RedactedBlot(Redacted): """ Used when a field is annotated to be blotted. """ def __repr__(self): return 'RedactedBlot(%r, %r, %r)' % (self.name, self.namespace, self.regex) class RedactedHash(Redacted): """ Used when a field is annotated to be hashed. """ def __repr__(self): return 'RedactedHash(%r, %r, %r)' % (self.name, self.namespace, self.regex) class CustomAnnotation(Annotation): """ Used when a field is annotated with a custom annotation type. """ def __init__(self, name, namespace, ast_node, annotation_type_name, annotation_type_ns, args, kwargs): super(CustomAnnotation, self).__init__(name, namespace, ast_node) self.annotation_type_name = annotation_type_name self.annotation_type_ns = annotation_type_ns self.args = args self.kwargs = kwargs self.annotation_type = None def set_attributes(self, annotation_type): self.annotation_type = annotation_type # check for too many parameters for args if len(self.args) > len(self.annotation_type.params): raise InvalidSpec('Too many parameters passed to annotation type %s' % (self.annotation_type.name), self._ast_node.lineno, self._ast_node.path) # check for unknown keyword arguments acceptable_param_names = set((param.name for param in self.annotation_type.params)) for param_name in self.kwargs: if param_name not in acceptable_param_names: raise InvalidSpec('Unknown parameter %s passed to annotation type %s' % (param_name, self.annotation_type.name), self._ast_node.lineno, self._ast_node.path) for i, param in enumerate(self.annotation_type.params): # first figure out and validate value for this param # arguments are either all kwargs or all args, so don't need to worry about # providing both positional and keyword argument for same parameter if param.name in self.kwargs or i < len(self.args): param_value = self.kwargs[param.name] if self.kwargs else self.args[i] try: param.data_type.check(param_value) except ValueError as e: raise InvalidSpec('Invalid value for parameter %s of annotation type %s: %s' % (param.name, self.annotation_type.name, e), self._ast_node.lineno, self._ast_node.path) elif isinstance(param.data_type, Nullable): param_value = None elif param.has_default: param_value = param.default else: raise InvalidSpec('No value specified for parameter %s of annotation type %s' % (param.name, self.annotation_type.name), self._ast_node.lineno, self._ast_node.path) # now set both kwargs and args to correct value so backend code generators can use # whichever is more convenient (like if kwargs are not supported in a language) self.kwargs[param.name] = param_value if i < len(self.args): self.args[i] = param_value else: self.args.append(param_value) class Alias(Composite): """ NOTE: The categorization of aliases as a composite type is arbitrary. It fit here better than as a primitive or user-defined type. """ def __init__(self, name, namespace, ast_node): """ When this is instantiated, the type is treated as a forward reference. Only when :meth:`set_attributes` is called is the type considered to be fully defined. :param str name: Name of type. :param stone.ir.ApiNamespace namespace: The namespace this type is defined in. :param ast_node: Raw type definition from the parser. :type ast_node: stone.frontend.ast.AstTypeDef """ super(Alias, self).__init__() self._name = name self.namespace = namespace self._ast_node = ast_node # Populated by :meth:`set_attributes` self.raw_doc = None self.doc = None self.data_type = None self.redactor = None self.custom_annotations = [] def set_annotations(self, annotations): for annotation in annotations: if isinstance(annotation, Redacted): # Make sure we don't set multiple conflicting annotations on one alias if self.redactor: raise InvalidSpec("Redactor already set as %r" % str(self.redactor), self._ast_node.lineno) self.redactor = annotation elif isinstance(annotation, CustomAnnotation): # Note: we don't need to do this for builtin annotations because # they are treated as globals at the IR level record_custom_annotation_imports(annotation, self.namespace) self.custom_annotations.append(annotation) else: raise InvalidSpec("Aliases only support 'Redacted' and custom annotations, not %r" % str(annotation), self._ast_node.lineno) def set_attributes(self, doc, data_type): """ :param Optional[str] doc: Documentation string of alias. :param data_type: The source data type referenced by the alias. """ self.raw_doc = doc self.doc = doc_unwrap(doc) self.data_type = data_type # Make sure we don't have a cyclic reference. # Since attributes are set one data type at a time, only the last data # type to be populated in a cycle will be able to detect the cycle. # Before that, the cycle will be broken by an alias with no populated # source. cur_data_type = data_type while is_alias(cur_data_type): cur_data_type = cur_data_type.data_type if cur_data_type == self: raise InvalidSpec( "Alias '%s' is part of a cycle." % self.name, self._ast_node.lineno, self._ast_node.path) @property def name(self): return self._name def check(self, val): return self.data_type.check(val) def check_example(self, ex_field): # TODO: Assert that this isn't a user-defined type. return self.data_type.check_example(ex_field) def _has_example(self, label): # TODO: Assert that this is a user-defined type return self.data_type._has_example(label) def _compute_example(self, label): return self.data_type._compute_example(label) def check_attr_repr(self, attr_field): return self.data_type.check_attr_repr(attr_field) def __repr__(self): return 'Alias(%r, %r)' % (self.name, self.data_type) def unwrap_nullable(data_type): """ Convenience method to unwrap Nullable from around a DataType. Args: data_type (DataType): The target to unwrap. Return: Tuple[DataType, bool]: The underlying data type and a bool indicating whether the input type was nullable. """ if is_nullable_type(data_type): return data_type.data_type, True else: return data_type, False def unwrap_aliases(data_type): """ Convenience method to unwrap all Alias(es) from around a DataType. Args: data_type (DataType): The target to unwrap. Return: Tuple[DataType, bool]: The underlying data type and a bool indicating whether the input type had at least one alias layer. """ unwrapped_alias = False while is_alias(data_type): unwrapped_alias = True data_type = data_type.data_type return data_type, unwrapped_alias def resolve_aliases(data_type): """ Resolve all chained / nested aliases. This will recursively point nested aliases to their resolved data type (first non-alias in the chain). Note: This differs from unwrap_alias which simply identifies/returns the resolved data type. Args: data_type (DataType): The target DataType/Alias to resolve. Return: DataType: The resolved type. """ if not is_alias(data_type): return data_type resolved = resolve_aliases(data_type.data_type) data_type.data_type = resolved return resolved def strip_alias(data_type): """ Strip alias from a data_type chain - this function should be used *after* aliases are resolved (see resolve_aliases fn): Loops through given data type chain (unwraps types), replaces first alias with underlying type, and then terminates. Note: Stops on encountering the first alias as it assumes intermediate aliases are already removed. Args: data_type (DataType): The target DataType chain to strip. Return: None """ while hasattr(data_type, 'data_type'): if is_alias(data_type.data_type): data_type.data_type = data_type.data_type.data_type break data_type = data_type.data_type def unwrap(data_type): """ Convenience method to unwrap all Aliases and Nullables from around a DataType. This checks for nullable wrapping aliases, as well as aliases wrapping nullables. Args: data_type (DataType): The target to unwrap. Return: Tuple[DataType, bool, bool]: The underlying data type; a bool that is set if a nullable was present; a bool that is set if an alias was present. """ unwrapped_nullable = False unwrapped_alias = False while is_alias(data_type) or is_nullable_type(data_type): if is_nullable_type(data_type): unwrapped_nullable = True if is_alias(data_type): unwrapped_alias = True data_type = data_type.data_type return data_type, unwrapped_nullable, unwrapped_alias def is_alias(data_type): return isinstance(data_type, Alias) def is_bytes_type(data_type): return isinstance(data_type, Bytes) def is_boolean_type(data_type): return isinstance(data_type, Boolean) def is_composite_type(data_type): return isinstance(data_type, Composite) def is_field_type(data_type): return isinstance(data_type, Field) def is_float_type(data_type): return isinstance(data_type, (Float32, Float64)) def is_integer_type(data_type): return isinstance(data_type, (UInt32, UInt64, Int32, Int64)) def is_list_type(data_type): return isinstance(data_type, List) def is_map_type(data_type): return isinstance(data_type, Map) def is_nullable_type(data_type): return isinstance(data_type, Nullable) def is_numeric_type(data_type): return is_integer_type(data_type) or is_float_type(data_type) def is_primitive_type(data_type): return isinstance(data_type, Primitive) def is_string_type(data_type): return isinstance(data_type, String) def is_struct_type(data_type): return isinstance(data_type, Struct) def is_tag_ref(val): return isinstance(val, TagRef) def is_timestamp_type(data_type): return isinstance(data_type, Timestamp) def is_union_type(data_type): return isinstance(data_type, Union) def is_user_defined_type(data_type): return isinstance(data_type, UserDefined) def is_void_type(data_type): return isinstance(data_type, Void) def is_int32_type(data_type): return isinstance(data_type, Int32) def is_int64_type(data_type): return isinstance(data_type, Int64) def is_uint32_type(data_type): return isinstance(data_type, UInt32) def is_uint64_type(data_type): return isinstance(data_type, UInt64) def is_float32_type(data_type): return isinstance(data_type, Float32) def is_float64_type(data_type): return isinstance(data_type, Float64) stone-3.3.1/stone/py.typed000066400000000000000000000000001417406541500154720ustar00rootroot00000000000000stone-3.3.1/stone/typing_hacks.py000066400000000000000000000003171417406541500170430ustar00rootroot00000000000000MYPY = False if MYPY: from typing import cast # noqa # pylint: disable=unused-import,useless-suppression,import-error else: def cast(typ, obj): # pylint: disable=unused-argument return obj stone-3.3.1/test/000077500000000000000000000000001417406541500136345ustar00rootroot00000000000000stone-3.3.1/test/__init__.py000066400000000000000000000000001417406541500157330ustar00rootroot00000000000000stone-3.3.1/test/backend_test_util.py000066400000000000000000000021171417406541500176720ustar00rootroot00000000000000MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression try: # Works for Py 3.3+ from unittest.mock import DEFAULT, Mock except ImportError: # See https://github.com/python/mypy/issues/1153#issuecomment-253842414 from mock import DEFAULT, Mock # type: ignore from stone.backend import Backend def _mock_output(backend): # type: (Backend) -> typing.Callable[[], str] """ Mock out Backend's .emit function, and return a list containing all params emit was called with. """ recorded_output = [] # type: typing.List[str] output_buffer_to_string = backend.output_buffer_to_string def record_output(): recorded_output.append(output_buffer_to_string()) return DEFAULT backend.output_buffer_to_string = Mock( # type: ignore wraps=output_buffer_to_string, side_effect=record_output) def get_output(): backend.output_buffer_to_string = output_buffer_to_string return recorded_output[0] if recorded_output else '' return get_output stone-3.3.1/test/requirements.txt000066400000000000000000000000471417406541500171210ustar00rootroot00000000000000mock>=2.0.0,<5.0 coverage==5.5 pytest<7stone-3.3.1/test/resources/000077500000000000000000000000001417406541500156465ustar00rootroot00000000000000stone-3.3.1/test/resources/typescript.template000066400000000000000000000000121417406541500216020ustar00rootroot00000000000000/*TYPES*/ stone-3.3.1/test/test_backend.py000077500000000000000000000431151417406541500166430ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals import unittest from stone.ir import ( Alias, Api, ApiNamespace, ApiRoute, List, Nullable, Boolean, String, Struct, StructField, Union, UnionField, resolve_aliases, strip_alias ) from stone.backend import ( remove_aliases_from_api, CodeBackend ) _MYPY = False if _MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression # Hack to get around some of Python 2's standard library modules that # accept ascii-encodable unicode literals in lieu of strs, but where # actually passing such literals results in errors with mypy --py2. See # and # . import importlib argparse = importlib.import_module(str('argparse')) # type: typing.Any class _Tester(CodeBackend): """A no-op backend used to test helper methods.""" def generate(self, api): pass class _TesterCmdline(CodeBackend): cmdline_parser = argparse.ArgumentParser() cmdline_parser.add_argument('-v', '--verbose', action='store_true') def generate(self, api): pass class TestBackend(unittest.TestCase): """ Tests the interface exposed to backends. """ def test_api_namespace(self): ns = ApiNamespace('files') a1 = Struct('A1', None, ns) a1.set_attributes(None, [StructField('f1', Boolean(), None, None)]) a2 = Struct('A2', None, ns) a2.set_attributes(None, [StructField('f2', Boolean(), None, None)]) l1 = List(a1) s = String() route = ApiRoute('test/route', 1, None) route.set_attributes(None, None, l1, a2, s, None) ns.add_route(route) # Test that only user-defined types are returned. route_io = ns.get_route_io_data_types() self.assertIn(a1, route_io) self.assertIn(a2, route_io) self.assertNotIn(l1, route_io) self.assertNotIn(s, route_io) def test_code_backend_helpers(self): t = _Tester(None, []) self.assertEqual(t.filter_out_none_valued_keys({}), {}) self.assertEqual(t.filter_out_none_valued_keys({'a': None}), {}) self.assertEqual(t.filter_out_none_valued_keys({'a': None, 'b': 3}), {'b': 3}) def test_code_backend_basic_emitters(self): t = _Tester(None, []) # Check basic emit t.emit('hello') self.assertEqual(t.output_buffer_to_string(), 'hello\n') t.clear_output_buffer() # Check that newlines are disallowed in emit self.assertRaises(AssertionError, lambda: t.emit('hello\n')) # Check indent context manager t.emit('hello') with t.indent(): t.emit('world') with t.indent(): t.emit('!') expected = """\ hello world ! """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() # -------------------------------------------------------- # Check text wrapping emitter with t.indent(): t.emit_wrapped_text('Colorless green ideas sleep furiously', prefix='$', initial_prefix='>', subsequent_prefix='|', width=13) expected = """\ $>Colorless $|green $|ideas $|sleep $|furiously """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() def test_code_backend_list_gen(self): t = _Tester(None, []) t.generate_multiline_list(['a=1', 'b=2']) expected = """\ (a=1, b=2) """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1', 'b=2'], 'def __init__', after=':') expected = """\ def __init__(a=1, b=2): """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1', 'b=2'], before='function_to_call', compact=False) expected = """\ function_to_call( a=1, b=2, ) """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1', 'b=2'], 'function_to_call', compact=False, skip_last_sep=True) expected = """\ function_to_call( a=1, b=2 ) """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1', 'b=2'], 'def func', ':', compact=False, skip_last_sep=True) expected = """\ def func( a=1, b=2 ): """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1'], 'function_to_call', compact=False) expected = 'function_to_call(a=1)\n' self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list(['a=1'], 'function_to_call', compact=True) expected = 'function_to_call(a=1)\n' self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list([], 'function_to_call', compact=False) expected = 'function_to_call()\n' self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() t.generate_multiline_list([], 'function_to_call', compact=True) expected = 'function_to_call()\n' self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() # Test delimiter t.generate_multiline_list(['String'], 'List', delim=('<', '>'), compact=True) expected = 'List\n' self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() def test_code_backend_block_gen(self): t = _Tester(None, []) with t.block('int sq(int x)', ';'): t.emit('return x*x;') expected = """\ int sq(int x) { return x*x; }; """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() with t.block('int sq(int x)', allman=True): t.emit('return x*x;') expected = """\ int sq(int x) { return x*x; } """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() with t.block('int sq(int x)', delim=('<', '>'), dent=8): t.emit('return x*x;') expected = """\ int sq(int x) < return x*x; > """ self.assertEqual(t.output_buffer_to_string(), expected) t.clear_output_buffer() def test_backend_cmdline(self): t = _TesterCmdline(None, ['-v']) self.assertTrue(t.args.verbose) def test_resolve_aliases(self): first_alias = Alias(None, None, None) first_alias.data_type = String() resolved_type = resolve_aliases(first_alias.data_type) # Test that single-level alias chain resolves self.assertIsInstance(resolved_type, String) self.assertIsInstance(first_alias.data_type, String) first_alias = Alias(None, None, None) second_alias = Alias(None, None, None) first_alias.data_type = second_alias second_alias.data_type = String() # Test that a two-level alias chain resolves resolved_type = resolve_aliases(first_alias.data_type) first_alias.data_type = resolved_type self.assertIsInstance(resolved_type, String) self.assertIsInstance(first_alias.data_type, String) self.assertIsInstance(second_alias.data_type, String) def test_strip_alias(self): first_alias = Alias(None, None, None) second_alias = Alias(None, None, None) third_alias = Alias(None, None, None) first_alias.data_type = second_alias second_alias.data_type = third_alias third_alias.data_type = String() test_struct = Struct('TestStruct', None, None) test_struct.set_attributes(None, [ StructField('field1', List(Nullable(first_alias)), None, None) ]) curr_type = first_alias while hasattr(curr_type, 'data_type'): curr_type.data_type = resolve_aliases(curr_type.data_type) curr_type = curr_type.data_type self.assertIsInstance(first_alias.data_type, String) self.assertEqual(len(test_struct.fields), 1) field = test_struct.fields[0] strip_alias(field.data_type) list_type = field.data_type self.assertIsInstance(list_type, List) nullable_type = list_type.data_type self.assertIsInstance(nullable_type, Nullable) string_type = nullable_type.data_type self.assertIsInstance(string_type, String) def test_preserve_aliases_from_api(self): api = Api(version=None) # Ensure imports come after 'preserve_alias' lexiographicaly # to catch namespace ordering bugs api.ensure_namespace('preserve_alias') api.ensure_namespace('zzzz') ns = api.namespaces['preserve_alias'] imported = api.namespaces['zzzz'] namespace_id = Alias('NamespaceId', ns, None) namespace_id.data_type = String() shared_folder_id = Alias('SharedFolderId', ns, None) shared_folder_id.set_attributes(None, namespace_id) path_root_id = Alias('PathRootId', ns, None) path_root_id.set_attributes(None, shared_folder_id) foo_alias = Alias('FooAlias', None, None) foo_alias.set_attributes(None, String()) bar_alias = Alias('BarAlias', None, None) bar_alias.set_attributes(None, foo_alias) ns.add_alias(namespace_id) ns.add_alias(shared_folder_id) ns.add_alias(path_root_id) imported.add_alias(foo_alias) imported.add_alias(bar_alias) test_struct = Struct('TestStruct', ns, None) test_struct.set_attributes(None, [StructField('field1', path_root_id, None, None)]) test_union = Union('TestUnion', ns, None, None) test_union.set_attributes(None, [UnionField('test', path_root_id, None, None)]) dependent_struct = Struct('DependentStruct', ns, None) dependent_struct.set_attributes(None, [ StructField('field_alias', imported.alias_by_name['BarAlias'], None, None) ]) ns.add_data_type(test_struct) ns.add_data_type(test_union) ns.add_data_type(dependent_struct) struct_alias = Alias('StructAlias', ns, None) struct_alias.set_attributes(None, test_struct) ns.add_alias(struct_alias) # Ensure namespace exists self.assertEqual(len(api.namespaces), 2) self.assertTrue('preserve_alias' in api.namespaces) self.assertTrue('zzzz' in api.namespaces) ns = api.namespaces['preserve_alias'] imported = api.namespaces['zzzz'] # Ensure aliases exist self.assertEqual(len(ns.aliases), 4) self.assertEqual(len(imported.aliases), 2) aliases = { alias._name: alias for alias in ns.aliases } imported_aliases = { alias.name: alias for alias in imported.aliases } data_types = { data_type._name: data_type for data_type in ns.data_types } # Ensure aliases are in the namespace self.assertTrue('NamespaceId' in aliases) self.assertTrue('SharedFolderId' in aliases) self.assertTrue('PathRootId' in aliases) self.assertTrue('StructAlias' in aliases) self.assertTrue('FooAlias' in imported_aliases) self.assertTrue('BarAlias' in imported_aliases) # Ensure aliases resolve to proper types self.assertIsInstance(aliases['NamespaceId'].data_type, String) self.assertIsInstance(aliases['SharedFolderId'].data_type, Alias) self.assertIsInstance(aliases['PathRootId'].data_type, Alias) self.assertIsInstance(aliases['StructAlias'].data_type, Struct) self.assertIsInstance(imported_aliases['FooAlias'].data_type, String) self.assertIsInstance(imported_aliases['BarAlias'].data_type, Alias) # Ensure struct and union field aliases resolve to proper types self.assertIsInstance(data_types['TestStruct'], Struct) test_struct = data_types.get('TestStruct') dependent_struct = data_types.get('DependentStruct') self.assertTrue(len(test_struct.fields), 1) self.assertTrue(len(dependent_struct.fields), 1) field = test_struct.fields[0] self.assertEqual(field.name, 'field1') self.assertIsInstance(field.data_type, Alias) field = dependent_struct.fields[0] self.assertEqual(field.name, 'field_alias') self.assertIsInstance(field.data_type, Alias) test_union = data_types['TestUnion'] self.assertTrue(len(test_union.fields), 1) field = test_union.fields[0] self.assertEqual(field.name, 'test') self.assertIsInstance(field.data_type, Alias) def test_no_preserve_aliases_from_api(self): api = Api(version=None) # Ensure imports come after 'preserve_alias' lexiographicaly # to catch namespace ordering bugs api.ensure_namespace('preserve_alias') api.ensure_namespace('zzzz') ns = api.namespaces['preserve_alias'] imported = api.namespaces['zzzz'] # Setup aliases namespace_id = Alias('NamespaceId', ns, None) namespace_id.data_type = String() shared_folder_id = Alias('SharedFolderId', ns, None) shared_folder_id.set_attributes(None, namespace_id) path_root_id = Alias('PathRootId', ns, None) path_root_id.set_attributes(None, shared_folder_id) nullable_alias = Alias('NullableAlias', ns, None) nullable_alias.set_attributes(None, Nullable(path_root_id)) foo_alias = Alias('FooAlias', None, None) foo_alias.set_attributes(None, String()) bar_alias = Alias('BarAlias', None, None) bar_alias.set_attributes(None, foo_alias) ns.add_alias(namespace_id) ns.add_alias(shared_folder_id) ns.add_alias(path_root_id) ns.add_alias(nullable_alias) imported.add_alias(foo_alias) imported.add_alias(bar_alias) # Setup composite types test_struct = Struct('TestStruct', ns, None) test_struct.set_attributes(None, [ StructField('field_alias', path_root_id, None, None), StructField('field_nullable_alias', nullable_alias, None, None), StructField('field_list_of_alias', List(path_root_id), None, None) ]) test_union = Union('TestUnion', ns, None, None) test_union.set_attributes(None, [UnionField('test', path_root_id, None, None)]) dependent_struct = Struct('DependentStruct', ns, None) dependent_struct.set_attributes(None, [ StructField('field_alias', imported.alias_by_name['BarAlias'], None, None) ]) ns.add_data_type(test_struct) ns.add_data_type(test_union) ns.add_data_type(dependent_struct) # Setup aliases on composite types struct_alias = Alias('StructAlias', ns, None) struct_alias.set_attributes(None, test_struct) ns.add_alias(struct_alias) api = remove_aliases_from_api(api) # Ensure namespace exists self.assertEqual(len(api.namespaces), 2) self.assertTrue('preserve_alias' in api.namespaces) self.assertTrue('zzzz' in api.namespaces) ns = api.namespaces['preserve_alias'] imported = api.namespaces['zzzz'] # Ensure aliases are gone self.assertEqual(len(ns.aliases), 0) self.assertEqual(len(imported.aliases), 0) data_types = { data_type._name: data_type for data_type in ns.data_types } # Ensure struct and union field aliases resolve to proper types test_struct = data_types.get('TestStruct') self.assertIsInstance(test_struct, Struct) self.assertEqual(len(test_struct.fields), 3) for field in test_struct.fields: if field.name == 'field_list_of_alias': self.assertIsInstance(field.data_type, List) list_type = field.data_type.data_type self.assertIsInstance(list_type, String) elif field.name == 'field_nullable_alias': field_type = field.data_type self.assertIsInstance(field_type, Nullable) self.assertIsInstance(field_type.data_type, String) else: self.assertIsInstance(field.data_type, String) test_union = data_types['TestUnion'] self.assertTrue(len(test_union.fields), 1) field = test_union.fields[0] self.assertEqual(field.name, 'test') self.assertIsInstance(field.data_type, String) # Ensure struct using imported alias resolves properly dependent_struct = data_types.get('DependentStruct') self.assertIsInstance(dependent_struct, Struct) self.assertEqual(len(dependent_struct.fields), 1) field = dependent_struct.fields[0] self.assertIsInstance(field.data_type, String) def test_route_hashing(self): route = ApiRoute('test/route', 1, None) same_route = ApiRoute("test/route", 1, None) diff_version = ApiRoute("test/route", 2, None) diff_route = ApiRoute("test/route_2", 1, None) self.assertEqual(hash(route), hash(same_route)) self.assertNotEqual(hash(route), hash(diff_version)) self.assertNotEqual(hash(route), hash(diff_route)) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_cli.py000077500000000000000000000126211417406541500160210ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals import unittest from stone.cli_helpers import parse_route_attr_filter class MockRoute(): """Used to test filtering on a route's attrs.""" def __init__(self, attrs): self.attrs = attrs class TestCLI(unittest.TestCase): def test_parse_route_attr_filter(self): _, errs = parse_route_attr_filter('*=3') self.assertNotEqual(len(errs), 0) _, errs = parse_route_attr_filter('test') self.assertEqual(len(errs), 1) self.assertEqual(errs[0], 'Unexpected end of expression.') _, errs = parse_route_attr_filter('hide=true)') self.assertNotEqual(len(errs), 0) _, errs = parse_route_attr_filter('(hide=true') self.assertNotEqual(len(errs), 0) _, errs = parse_route_attr_filter('hide=true and and size=1') self.assertNotEqual(len(errs), 0) # Test bool expr, errs = parse_route_attr_filter('hide=true') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'hide': True}))) self.assertFalse(expr.eval(MockRoute({'hide': 'true'}))) # Test int expr, errs = parse_route_attr_filter('level=1') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'level': 1}))) self.assertFalse(expr.eval(MockRoute({'level': 2}))) self.assertFalse(expr.eval(MockRoute({'level': '1'}))) self.assertFalse(expr.eval(MockRoute({}))) # Test float expr, errs = parse_route_attr_filter('f=1.25') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'f': 1.25}))) self.assertFalse(expr.eval(MockRoute({'f': 3}))) self.assertFalse(expr.eval(MockRoute({'f': '1.25'}))) self.assertFalse(expr.eval(MockRoute({}))) # Test string expr, errs = parse_route_attr_filter('status="alpha"') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'status': 'alpha'}))) self.assertFalse(expr.eval(MockRoute({'status': 'beta'}))) self.assertFalse(expr.eval(MockRoute({'status': 0}))) self.assertFalse(expr.eval(MockRoute({}))) # Test null expr, errs = parse_route_attr_filter('status=null') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'status': None}))) self.assertFalse(expr.eval(MockRoute({'status': 'beta'}))) self.assertFalse(expr.eval(MockRoute({'status': 0}))) self.assertTrue(expr.eval(MockRoute({}))) # Test conjunction: or expr, errs = parse_route_attr_filter('a=1 or b=1') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1}))) self.assertTrue(expr.eval(MockRoute({'b': 1}))) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 1}))) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 10}))) self.assertFalse(expr.eval(MockRoute({'a': '0', 'b': 0}))) self.assertFalse(expr.eval(MockRoute({'a': 0}))) self.assertFalse(expr.eval(MockRoute({}))) # Test conjunction: and expr, errs = parse_route_attr_filter('a=1 and b=1') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 1}))) self.assertFalse(expr.eval(MockRoute({'a': 1}))) self.assertFalse(expr.eval(MockRoute({'b': 1}))) self.assertFalse(expr.eval(MockRoute({'a': 1, 'b': 10}))) self.assertFalse(expr.eval(MockRoute({'a': '0', 'b': 0}))) self.assertFalse(expr.eval(MockRoute({'a': 0}))) self.assertFalse(expr.eval(MockRoute({}))) # Test multiple conjunctions expr, errs = parse_route_attr_filter('a=1 or a=2 or a=3') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1}))) self.assertTrue(expr.eval(MockRoute({'a': 2}))) self.assertTrue(expr.eval(MockRoute({'a': 3}))) self.assertFalse(expr.eval(MockRoute({'a': 4}))) # Test "and" has higher precendence than "or" expr, errs = parse_route_attr_filter('a=1 or a=2 and b=3 and c=4') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1}))) self.assertFalse(expr.eval(MockRoute({'a': 2}))) self.assertTrue(expr.eval(MockRoute({'a': 2, 'b': 3, 'c': 4}))) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 3, 'c': 4}))) self.assertFalse(expr.eval(MockRoute({'a': 0, 'b': 3, 'c': 4}))) expr, errs = parse_route_attr_filter('a=2 and b=3 and c=4 or a=1') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1}))) self.assertFalse(expr.eval(MockRoute({'a': 2}))) self.assertTrue(expr.eval(MockRoute({'a': 2, 'b': 3, 'c': 4}))) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 3, 'c': 4}))) self.assertFalse(expr.eval(MockRoute({'a': 0, 'b': 3, 'c': 4}))) # Test parentheses for overriding precedence expr, errs = parse_route_attr_filter('(a=1 or a=2) and b=3 and c=4') self.assertEqual(len(errs), 0) self.assertTrue(expr.eval(MockRoute({'a': 1, 'b': 3, 'c': 4}))) self.assertTrue(expr.eval(MockRoute({'a': 2, 'b': 3, 'c': 4}))) self.assertFalse(expr.eval(MockRoute({'a': 1}))) self.assertFalse(expr.eval(MockRoute({'a': 1, 'b': 3}))) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_js_client.py000066400000000000000000000117121417406541500172210ustar00rootroot00000000000000import textwrap import unittest from stone.backends.js_client import JavascriptClientBackend from test.backend_test_util import _mock_output from stone.ir import Api, ApiNamespace, ApiRoute, Void, Int32 from stone.ir.data_types import Struct MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class TestGeneratedJSClient(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGeneratedJSClient, self).__init__(*args, **kwargs) def _get_api(self): # type () -> Api api = Api(version='0.1b1') api.route_schema = Struct('Route', 'stone_cfg', None) route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata`', Void(), Void(), Void(), {}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, ':route:`get_metadata:2`', Void(), Int32(), Void(), {}) route3 = ApiRoute('get_metadata', 3, None) route3.set_attributes(None, ':route:`get_metadata:3`', Int32(), Int32(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) ns.add_route(route3) api.namespaces[ns.name] = ns return api, ns def test_route_versions(self): # type: () -> None api, _ = self._get_api() backend = JavascriptClientBackend( target_folder_path='output', args=['files', '-c', 'DropboxBase']) get_result = _mock_output(backend) backend.generate(api) result = get_result() expected = textwrap.dedent('''\ // Auto-generated by Stone, do not modify. var routes = {}; /** * get_metadata * @function DropboxBase#filesGetMetadata * @returns {Promise.>} */ routes.filesGetMetadata = function () { return this.request("files/get_metadata", null); }; /** * get_metadata_v2 * @function DropboxBase#filesGetMetadataV2 * @returns {Promise.>} */ routes.filesGetMetadataV2 = function () { return this.request("files/get_metadata_v2", null); }; /** * get_metadata_v3 * @function DropboxBase#filesGetMetadataV3 * @arg {number} arg - The request parameters. * @returns {Promise.>} */ routes.filesGetMetadataV3 = function (arg) { return this.request("files/get_metadata_v3", arg); }; export { routes }; ''') assert result == expected def test_wrap_response_in_flag(self): # type: () -> None api, _ = self._get_api() backend = JavascriptClientBackend( target_folder_path='output', args=['files', '-c', 'DropboxBase', '--wrap-response-in', 'DropboxResponse']) get_result = _mock_output(backend) backend.generate(api) result = get_result() expected = textwrap.dedent('''\ // Auto-generated by Stone, do not modify. var routes = {}; /** * get_metadata * @function DropboxBase#filesGetMetadata * @returns {Promise., Error.>} */ routes.filesGetMetadata = function () { return this.request("files/get_metadata", null); }; /** * get_metadata_v2 * @function DropboxBase#filesGetMetadataV2 * @returns {Promise., Error.>} */ routes.filesGetMetadataV2 = function () { return this.request("files/get_metadata_v2", null); }; /** * get_metadata_v3 * @function DropboxBase#filesGetMetadataV3 * @arg {number} arg - The request parameters. * @returns {Promise., Error.>} */ routes.filesGetMetadataV3 = function (arg) { return this.request("files/get_metadata_v3", arg); }; export { routes }; ''') assert result == expected def test_route_with_version_number_conflict(self): # type: () -> None api, ns = self._get_api() # Add a conflicting route route3 = ApiRoute('get_metadata_v2', 1, None) route3.set_attributes(None, None, Void(), Int32(), Void(), {}) ns.add_route(route3) backend = JavascriptClientBackend( target_folder_path='output', args=['files', '-c', 'DropboxBase']) with self.assertRaises(RuntimeError) as cm: backend.generate(api) self.assertTrue(str(cm.exception).startswith( 'There is a name conflict between')) stone-3.3.1/test/test_python_client.py000066400000000000000000000264001417406541500201260ustar00rootroot00000000000000import textwrap from stone.backends.python_client import PythonClientBackend from stone.ir import ( ApiNamespace, ApiRoute, Int32, List, Map, Nullable, String, Void, StructField, Struct, ) MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression import unittest class TestGeneratedPythonClient(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGeneratedPythonClient, self).__init__(*args, **kwargs) def _evaluate_namespace(self, ns): # type: (ApiNamespace) -> typing.Text backend = PythonClientBackend( target_folder_path='output', args=['-a', 'scope', '-a', 'another_attribute', '-m', 'files', '-c', 'DropboxBase', '-t', 'dropbox']) backend._generate_routes(ns) return backend.output_buffer_to_string() def _evaluate_namespace_with_auth_mode(self, ns, auth_mode): # type: (ApiNamespace, str) -> typing.Text # supply supported auth modes to the SDK generator using the new syntax backend = PythonClientBackend( target_folder_path='output', args=['-w', auth_mode, '-m', 'files', '-c', 'DropboxBase', '-t', 'dropbox']) backend._generate_route_methods({ns}) return backend.output_buffer_to_string() def test_route_with_version_number(self): # type: () -> None route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata:2`', Void(), Void(), Void(), {}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, None, Void(), Int32(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) result = self._evaluate_namespace(ns) expected = textwrap.dedent('''\ def files_get_metadata(self): """ :meth:`files_get_metadata_v2` :rtype: None """ arg = None r = self.request( files.get_metadata, 'files', arg, None, ) return None def files_get_metadata_v2(self): arg = None r = self.request( files.get_metadata_v2, 'files', arg, None, ) return r ''') self.assertEqual(result, expected) def test_route_with_auth_mode1(self): # type: () -> None route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata:2`', Void(), Void(), Void(), {'auth': 'app'}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, None, Void(), Int32(), Void(), {'auth': 'user, app'}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) result = self._evaluate_namespace_with_auth_mode(ns, 'user') expected = textwrap.dedent('''\ # ------------------------------------------ # Routes in files namespace def files_get_metadata_v2(self): arg = None r = self.request( files.get_metadata_v2, 'files', arg, None, ) return r ''') self.assertEqual(result, expected) def test_route_with_auth_mode2(self): # type: () -> None route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata:2`', Void(), Void(), Void(), {'auth': 'user'}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, None, Void(), Int32(), Void(), {'auth': 'user, app'}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) result = self._evaluate_namespace_with_auth_mode(ns, 'user') expected = textwrap.dedent('''\ # ------------------------------------------ # Routes in files namespace def files_get_metadata(self): """ :meth:`files_get_metadata_v2` :rtype: None """ arg = None r = self.request( files.get_metadata, 'files', arg, None, ) return None def files_get_metadata_v2(self): arg = None r = self.request( files.get_metadata_v2, 'files', arg, None, ) return r ''') self.assertEqual(result, expected) def test_route_with_auth_mode3(self): # type: () -> None route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata:2`', Void(), Void(), Void(), {'auth': 'app'}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, None, Void(), Int32(), Void(), {'auth': 'app, team'}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) result = self._evaluate_namespace_with_auth_mode(ns, 'user') expected = textwrap.dedent('''\ # ------------------------------------------ # Routes in files namespace ''') self.assertEqual(result, expected) def test_route_with_version_number_name_conflict(self): # type: () -> None route1 = ApiRoute('get_metadata', 2, None) route1.set_attributes(None, None, Void(), Int32(), Void(), {}) route2 = ApiRoute('get_metadata_v2', 1, None) route2.set_attributes(None, None, Void(), Void(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) with self.assertRaises(RuntimeError) as cm: self._evaluate_namespace(ns) self.assertEqual( 'There is a name conflict between {!r} and {!r}'.format(route1, route2), str(cm.exception)) def test_route_argument_doc_string(self): backend = PythonClientBackend( target_folder_path='output', args=['-m', 'files', '-c', 'DropboxBase', '-t', 'dropbox']) ns = ApiNamespace('files') self.assertEqual(backend._format_type_in_doc(ns, Int32()), 'int') self.assertEqual(backend._format_type_in_doc(ns, Void()), 'None') self.assertEqual(backend._format_type_in_doc(ns, List(String())), 'List[str]') self.assertEqual(backend._format_type_in_doc(ns, Nullable(String())), 'Nullable[str]') self.assertEqual(backend._format_type_in_doc(ns, Map(String(), Int32())), 'Map[str, int]') def test_route_with_attributes_in_docstring(self): # type: () -> None route = ApiRoute('get_metadata', 1, None) route.set_attributes(None, None, Void(), Void(), Void(), { 'scope': 'events.read', 'another_attribute': 'foo' }) ns = ApiNamespace('files') ns.add_route(route) result = self._evaluate_namespace(ns) expected = textwrap.dedent('''\ def files_get_metadata(self): """ Route attributes: scope: events.read another_attribute: foo :rtype: None """ arg = None r = self.request( files.get_metadata, 'files', arg, None, ) return None ''') self.assertEqual(result, expected) def test_route_with_none_attribute_in_docstring(self): # type: () -> None route = ApiRoute('get_metadata', 1, None) route.set_attributes(None, None, Void(), Void(), Void(), { 'scope': 'events.read', 'another_attribute': None }) ns = ApiNamespace('files') ns.add_route(route) result = self._evaluate_namespace(ns) expected = textwrap.dedent('''\ def files_get_metadata(self): """ Route attributes: scope: events.read :rtype: None """ arg = None r = self.request( files.get_metadata, 'files', arg, None, ) return None ''') self.assertEqual(result, expected) def test_route_with_attributes_and_doc_in_docstring(self): # type: () -> None """ In particular make sure there's spacing b/w overview and attrs. """ route = ApiRoute('get_metadata', 1, None) route.set_attributes(None, "Test string.", Void(), Void(), Void(), {'scope': 'events.read'}) ns = ApiNamespace('files') ns.add_route(route) result = self._evaluate_namespace(ns) expected = textwrap.dedent('''\ def files_get_metadata(self): """ Test string. Route attributes: scope: events.read :rtype: None """ arg = None r = self.request( files.get_metadata, 'files', arg, None, ) return None ''') self.assertEqual(result, expected) def test_route_with_doc_and_attribute_and_data_types(self): # type: () -> None ns = ApiNamespace('files') struct = Struct('MyStruct', ns, None) struct.set_attributes(None, [ StructField('field1', Int32(), None, None), StructField('field2', Int32(), None, None), ]) route = ApiRoute('test/route', 1, None) route.set_attributes( None, "Test string.", struct, Int32(), Void(), {'scope': 'events.read'} ) ns.add_route(route) result = self._evaluate_namespace(ns) expected = textwrap.dedent('''\ def files_test_route(self, field1, field2): """ Test string. Route attributes: scope: events.read :type field1: int :type field2: int :rtype: int """ arg = files.MyStruct(field1, field2) r = self.request( files.test_route, 'files', arg, None, ) return r ''') self.assertEqual(result, expected) # TODO: add more unit tests for client code generation stone-3.3.1/test/test_python_gen.py000077500000000000000000003435751417406541500174430ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals import base64 import datetime import importlib import json import shutil import six import subprocess import sys import unittest import stone.backends.python_rsrc.stone_base as bb import stone.backends.python_rsrc.stone_serializers as ss import stone.backends.python_rsrc.stone_validators as bv from stone.backends.python_rsrc.stone_serializers import ( CallerPermissionsInterface, json_encode, json_decode, _strftime as stone_strftime, ) class TestDropInModules(unittest.TestCase): """ Tests the stone_serializers and stone_validators modules. """ def mk_validator_testers(self, validator): def p(i): validator.validate(i) def f(i): self.assertRaises(bv.ValidationError, validator.validate, i) return p, f # 'p(input)' if you expect it to pass, 'f(input)' if you expect it to fail. def test_string_validator(self): s = bv.String(min_length=1, max_length=5, pattern='[A-z]+') # Not a string self.assertRaises(bv.ValidationError, lambda: s.validate(1)) # Too short self.assertRaises(bv.ValidationError, lambda: s.validate('')) # Too long self.assertRaises(bv.ValidationError, lambda: s.validate('a' * 6)) # Doesn't pass regex self.assertRaises(bv.ValidationError, lambda: s.validate('#')) # Passes s.validate('a') # Check that the validator is converting all strings to unicode self.assertIsInstance(s.validate('a'), six.text_type) def test_string_regex_anchoring(self): p, f = self.mk_validator_testers(bv.String(pattern=r'abc|xyz')) p('abc') p('xyz') f('_abc') f('abc_') f('_xyz') f('xyz_') def test_boolean_validator(self): b = bv.Boolean() b.validate(True) b.validate(False) self.assertRaises(bv.ValidationError, lambda: b.validate(1)) def test_integer_validator(self): i = bv.UInt32(min_value=10, max_value=100) # Not an integer self.assertRaises(bv.ValidationError, lambda: i.validate(1.4)) # Too small self.assertRaises(bv.ValidationError, lambda: i.validate(1)) # Too large self.assertRaises(bv.ValidationError, lambda: i.validate(101)) # Passes i.validate(50) # min_value is less than the default for the type self.assertRaises(AssertionError, lambda: bv.UInt32(min_value=-3)) # non-sensical min_value self.assertRaises(AssertionError, lambda: bv.UInt32(min_value=1.3)) def test_float_validator(self): f64 = bv.Float64() # Too large for a float to represent self.assertRaises(bv.ValidationError, lambda: f64.validate(10**310)) # inf and nan should be rejected self.assertRaises(bv.ValidationError, lambda: f64.validate(float('nan'))) self.assertRaises(bv.ValidationError, lambda: f64.validate(float('inf'))) # Passes f64.validate(1.1 * 10**300) # Test a float64 with an additional bound f64b = bv.Float64(min_value=0, max_value=100) # Check bounds self.assertRaises(bv.ValidationError, lambda: f64b.validate(1000)) self.assertRaises(bv.ValidationError, lambda: f64b.validate(-1)) # Test a float64 with an invalid bound self.assertRaises(AssertionError, lambda: bv.Float64(min_value=0, max_value=10**330)) f32 = bv.Float32() self.assertRaises(bv.ValidationError, lambda: f32.validate(3.5 * 10**38)) self.assertRaises(bv.ValidationError, lambda: f32.validate(-3.5 * 10**38)) # Passes f32.validate(0) def test_bytes_validator(self): b = bv.Bytes(min_length=1, max_length=10) # Not a valid binary type self.assertRaises(bv.ValidationError, lambda: b.validate('asdf')) # Too short self.assertRaises(bv.ValidationError, lambda: b.validate(b'')) # Too long self.assertRaises(bv.ValidationError, lambda: b.validate(b'\x00' * 11)) # Passes b.validate(b'\x00') def test_timestamp_validator(self): class UTC(datetime.tzinfo): def utcoffset(self, dt): # pylint: disable=unused-argument,useless-suppression return datetime.timedelta(0) def tzname(self, dt): # pylint: disable=unused-argument,useless-suppression return 'UTC' def dst(self, dt): # pylint: disable=unused-argument,useless-suppression return datetime.timedelta(0) class PST(datetime.tzinfo): def utcoffset(self, dt): # pylint: disable=unused-argument,useless-suppression return datetime.timedelta(-8) def tzname(self, dt): # pylint: disable=unused-argument,useless-suppression return 'PST' def dst(self, dt): # pylint: disable=unused-argument,useless-suppression return datetime.timedelta(0) t = bv.Timestamp('%a, %d %b %Y %H:%M:%S +0000') self.assertRaises(bv.ValidationError, lambda: t.validate('abcd')) now = datetime.datetime.utcnow() t.validate(now) then = datetime.datetime(1776, 7, 4, 12, 0, 0) t.validate(then) new_then = json_decode(t, json_encode(t, then)) self.assertEqual(then, new_then) # Accept a tzinfo only if it's UTC t.validate(now.replace(tzinfo=UTC())) # Do not accept a non-UTC tzinfo self.assertRaises(bv.ValidationError, lambda: t.validate(now.replace(tzinfo=PST()))) def test_list_validator(self): l1 = bv.List(bv.String(), min_items=1, max_items=10) # Not a valid list type self.assertRaises(bv.ValidationError, lambda: l1.validate('a')) # Too short self.assertRaises(bv.ValidationError, lambda: l1.validate([])) # Too long self.assertRaises(bv.ValidationError, lambda: l1.validate([1] * 11)) # Not a valid string type self.assertRaises(bv.ValidationError, lambda: l1.validate([1])) # Passes l1.validate(['a']) def test_map_validator(self): m = bv.Map(bv.String(pattern="^foo.*"), bv.String(pattern=".*bar$")) # applies validators of children m.validate({"foo-one": "one-bar", "foo-two": "two-bar"}) # does not match regex self.assertRaises(bv.ValidationError, lambda: m.validate({"one": "two"})) # does not match declared types self.assertRaises(bv.ValidationError, lambda: m.validate({1: 2})) def test_nullable_validator(self): n = bv.Nullable(bv.String()) # Absent case n.validate(None) # Fails string validation self.assertRaises(bv.ValidationError, lambda: n.validate(123)) # Passes n.validate('abc') # Stacking nullables isn't supported by our JSON wire format self.assertRaises(AssertionError, lambda: bv.Nullable(bv.Nullable(bv.String()))) self.assertRaises(AssertionError, lambda: bv.Nullable(bv.Void())) def test_void_validator(self): v = bv.Void() # Passes: Only case that validates v.validate(None) # Fails validation self.assertRaises(bv.ValidationError, lambda: v.validate(123)) def test_struct_validator(self): class C(object): _all_field_names_ = {'f'} _all_fields_ = [('f', bv.String())] f = None s = bv.Struct(C) self.assertRaises(bv.ValidationError, lambda: s.validate(object())) def test_json_encoder(self): self.assertEqual(json_encode(bv.Void(), None), json.dumps(None)) self.assertEqual(json_encode(bv.String(), 'abc'), json.dumps('abc')) self.assertEqual(json_encode(bv.String(), '\u2650'), json.dumps('\u2650')) self.assertEqual(json_encode(bv.UInt32(), 123), json.dumps(123)) # Because a bool is a subclass of an int, ensure they aren't mistakenly # encoded as a true/false in JSON when an integer is the data type. self.assertEqual(json_encode(bv.UInt32(), True), json.dumps(1)) self.assertEqual(json_encode(bv.Boolean(), True), json.dumps(True)) f = '%a, %d %b %Y %H:%M:%S +0000' now = datetime.datetime.utcnow() self.assertEqual(json_encode(bv.Timestamp('%a, %d %b %Y %H:%M:%S +0000'), now), json.dumps(now.strftime(f))) b = b'\xff' * 5 self.assertEqual(json_encode(bv.Bytes(), b), json.dumps(base64.b64encode(b).decode('ascii'))) self.assertEqual(json_encode(bv.Nullable(bv.String()), None), json.dumps(None)) self.assertEqual(json_encode(bv.Nullable(bv.String()), 'abc'), json.dumps('abc')) def test_json_encoder_union(self): class S(object): _all_field_names_ = {'f'} _all_fields_ = [('f', bv.String())] _f_value = bb.NOT_SET class U(object): # pylint: disable=no-member _tagmap = {'a': bv.Int64(), 'b': bv.Void(), 'c': bv.Struct(S), 'd': bv.List(bv.Int64()), 'e': bv.Nullable(bv.Int64()), 'f': bv.Nullable(bv.Struct(S)), 'g': bv.Map(bv.String(), bv.String())} _tag = None def __init__(self, tag, value=None): self._tag = tag self._value = value def get_a(self): return self._a def get_c(self): return self._c def get_d(self): return self._d @classmethod def _is_tag_present(cls, tag, cp): assert cp if tag in cls._tagmap: return True return False @classmethod def _get_val_data_type(cls, tag, cp): assert cp return cls._tagmap[tag] U.b = U('b') # Test primitive variant u = U('a', 64) self.assertEqual(json_encode(bv.Union(U), u, old_style=True), json.dumps({'a': 64})) # Test symbol variant u = U('b') self.assertEqual(json_encode(bv.Union(U), u, old_style=True), json.dumps('b')) # Test struct variant c = S() c.f = 'hello' c._f_value = c.f u = U('c', c) self.assertEqual(json_encode(bv.Union(U), u, old_style=True), json.dumps({'c': {'f': 'hello'}})) # Test list variant u = U('d', [1, 2, 3, 'a']) # lists should be re-validated during serialization self.assertRaises(bv.ValidationError, lambda: json_encode(bv.Union(U), u)) l1 = [1, 2, 3, 4] u = U('d', [1, 2, 3, 4]) self.assertEqual(json_encode(bv.Union(U), u, old_style=True), json.dumps({'d': l1})) # Test a nullable union self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), None), json.dumps(None)) self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), u, old_style=True), json.dumps({'d': l1})) # Test nullable primitive variant u = U('e', None) self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), u, old_style=True), json.dumps('e')) u = U('e', 64) self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), u, old_style=True), json.dumps({'e': 64})) # Test nullable composite variant u = U('f', None) self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), u, old_style=True), json.dumps('f')) u = U('f', c) self.assertEqual(json_encode(bv.Nullable(bv.Union(U)), u, old_style=True), json.dumps({'f': {'f': 'hello'}})) u = U('g', {'one': 2}) self.assertRaises(bv.ValidationError, lambda: json_encode(bv.Union(U), u)) m = {'one': 'two'} u = U('g', m) self.assertEqual(json_encode(bv.Union(U), u, old_style=True), json.dumps({'g': m})) def test_json_encoder_error_messages(self): class S3(object): _all_field_names_ = {'j'} _all_fields_ = [('j', bv.UInt64(max_value=10))] _j_value = bb.NOT_SET class S2(object): _all_field_names_ = {'i'} _all_fields_ = [('i', bv.Struct(S3))] _i_value = bb.NOT_SET class S(object): _all_field_names_ = {'f'} _all_fields_ = [('f', bv.Struct(S2))] _f_value = bb.NOT_SET class U(object): # pylint: disable=no-member _tagmap = {'t': bv.Nullable(bv.Struct(S))} _tag = None _catch_all = None def __init__(self, tag, value=None): self._tag = tag self._value = value def get_t(self): return self._t @classmethod def _is_tag_present(cls, tag, cp): assert cp if tag in cls._tagmap: return True return False @classmethod def _get_val_data_type(cls, tag, cp): assert cp return cls._tagmap[tag] s = S() s.f = S2() s._f_value = s.f s.f.i = S3() s.f._i_value = s.f.i # Test that validation error references outer and inner struct with self.assertRaises(bv.ValidationError): try: json_encode(bv.Struct(S), s) except bv.ValidationError as e: prefix = 'f.i: ' self.assertEqual(prefix, str(e)[:len(prefix)]) raise u = U('t', s) # Test that validation error references outer union and inner structs with self.assertRaises(bv.ValidationError): try: json_encode(bv.Union(U), u) except bv.ValidationError as e: prefix = 't.f.i: ' self.assertEqual(prefix, str(e)[:len(prefix)]) raise def test_json_decoder(self): self.assertEqual(json_decode(bv.String(), json.dumps('abc')), 'abc') self.assertRaises(bv.ValidationError, lambda: json_decode(bv.String(), json.dumps(32))) self.assertEqual(json_decode(bv.UInt32(), json.dumps(123)), 123) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.UInt32(), json.dumps('hello'))) self.assertEqual(json_decode(bv.Boolean(), json.dumps(True)), True) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Boolean(), json.dumps(1))) f = '%a, %d %b %Y %H:%M:%S +0000' now = datetime.datetime.utcnow().replace(microsecond=0) self.assertEqual(json_decode(bv.Timestamp('%a, %d %b %Y %H:%M:%S +0000'), json.dumps(now.strftime(f))), now) # Try decoding timestamp with bad type self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Timestamp('%a, %d %b %Y %H:%M:%S +0000'), '1')) b = b'\xff' * 5 self.assertEqual(json_decode(bv.Bytes(), json.dumps(base64.b64encode(b).decode('ascii'))), b) self.assertRaises(bv.ValidationError, json_decode, bv.Bytes(), json.dumps('=non-base64=')) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Bytes(), json.dumps(1))) self.assertEqual(json_decode(bv.Nullable(bv.String()), json.dumps(None)), None) self.assertEqual(json_decode(bv.Nullable(bv.String()), json.dumps('abc')), 'abc') self.assertEqual(json_decode(bv.Void(), json.dumps(None)), None) # Check that void can take any input if strict is False. self.assertEqual(json_decode(bv.Void(), json.dumps(12345), strict=False), None) # Check that an error is raised if strict is True and there's a non-null value self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Void(), json.dumps(12345), strict=True)) def test_json_decoder_struct(self): class S(object): _all_field_names_ = {'f', 'g'} _all_fields_ = [('f', bv.String()), ('g', bv.Nullable(bv.String()))] _g = None @property def f(self): return self._f @f.setter def f(self, val): self._f = val @property def g(self): return self._g @g.setter def g(self, val): self._g = val # Required struct fields must be present self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Struct(S), json.dumps({}))) json_decode(bv.Struct(S), json.dumps({'f': 't'})) # Struct fields can have null values for nullable fields msg = json.dumps({'f': 't', 'g': None}) json_decode(bv.Struct(S), msg) # Unknown struct fields raise error if strict msg = json.dumps({'f': 't', 'z': 123}) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Struct(S), msg, strict=True)) json_decode(bv.Struct(S), msg, strict=False) def test_json_decoder_union(self): class S(object): _all_field_names_ = {'f'} _all_fields_ = [('f', bv.String())] class U(object): _tagmap = {'a': bv.Int64(), 'b': bv.Void(), 'c': bv.Struct(S), 'd': bv.List(bv.Int64()), 'e': bv.Nullable(bv.Int64()), 'f': bv.Nullable(bv.Struct(S)), 'g': bv.Void(), 'h': bv.Map(bv.String(), bv.String())} _catch_all = 'g' _tag = None def __init__(self, tag, value=None): self._tag = tag self._value = value def get_a(self): return self._value def get_c(self): return self._value def get_d(self): return self._value @classmethod def _is_tag_present(cls, tag, cp): assert cp if tag in cls._tagmap: return True return False @classmethod def _get_val_data_type(cls, tag, cp): assert cp return cls._tagmap[tag] U.b = U('b') # TODO: When run with Python 3, pylint thinks `u` is a `datetime` # object. This results in spurious `no-member` errors, which we # choose to ignore here (for now). This does not happen when # running pylint with Python 2 (hence the useless-suppression). # pylint: disable=no-member,useless-suppression # Test primitive variant u = json_decode(bv.Union(U), json.dumps({'a': 64}), old_style=True) self.assertEqual(u.get_a(), 64) # Test void variant u = json_decode(bv.Union(U), json.dumps('b')) self.assertEqual(u._tag, 'b') self.assertIsInstance(u._tag, str) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Union(U), json.dumps({'b': [1, 2]}))) u = json_decode(bv.Union(U), json.dumps({'b': [1, 2]}), strict=False, old_style=True) self.assertEqual(u._tag, 'b') self.assertIsInstance(u._tag, str) # Test struct variant u = json_decode(bv.Union(U), json.dumps({'c': {'f': 'hello'}}), old_style=True) self.assertEqual(u.get_c().f, 'hello') self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Union(U), json.dumps({'c': [1, 2, 3]}))) # Test list variant l1 = [1, 2, 3, 4] u = json_decode(bv.Union(U), json.dumps({'d': l1}), old_style=True) self.assertEqual(u.get_d(), l1) # Test map variant m = {'one': 'two', 'three': 'four'} u = json_decode(bv.Union(U), json.dumps({'h': m}), old_style=True) self.assertEqual(u.get_d(), m) # Raises if unknown tag self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Union(U), json.dumps('z'))) # Unknown variant (strict=True) self.assertRaises(bv.ValidationError, lambda: json_decode(bv.Union(U), json.dumps({'z': 'test'}))) # Test catch all variant u = json_decode(bv.Union(U), json.dumps({'z': 'test'}), strict=False, old_style=True) self.assertEqual(u._tag, 'g') self.assertIsInstance(u._tag, str) # Test nullable union u = json_decode(bv.Nullable(bv.Union(U)), json.dumps(None), strict=False, old_style=True) self.assertEqual(u, None) # Test nullable union member u = json_decode(bv.Union(U), json.dumps('e')) self.assertEqual(u._tag, 'e') self.assertIsInstance(u._tag, str) self.assertEqual(u._value, None) u = json_decode(bv.Union(U), json.dumps({'e': 64}), strict=False, old_style=True) self.assertEqual(u._tag, 'e') self.assertIsInstance(u._tag, str) self.assertEqual(u._value, 64) # Test nullable composite variant u = json_decode(bv.Union(U), json.dumps('f')) self.assertEqual(u._tag, 'f') self.assertIsInstance(u._tag, str) u = json_decode(bv.Union(U), json.dumps({'f': {'f': 'hello'}}), strict=False, old_style=True) self.assertEqual(type(u._value), S) self.assertEqual(u._value.f, 'hello') def test_json_decoder_error_messages(self): class S3(object): _all_field_names_ = {'j'} _all_fields_ = [('j', bv.UInt64(max_value=10))] class S2(object): _all_field_names_ = {'i'} _all_fields_ = [('i', bv.Struct(S3))] class S(object): _all_field_names_ = {'f'} _all_fields_ = [('f', bv.Struct(S2))] class U(object): _tagmap = {'t': bv.Nullable(bv.Struct(S))} _tag = None _catch_all = None def __init__(self, tag, value=None): self._tag = tag self._value = value def get_t(self): return self._value @classmethod def _is_tag_present(cls, tag, cp): assert cp if tag in cls._tagmap: return True return False @classmethod def _get_val_data_type(cls, tag, cp): assert cp return cls._tagmap[tag] # Test that validation error references outer and inner struct with self.assertRaises(bv.ValidationError): try: json_decode(bv.Struct(S), json.dumps({'f': {'i': {}}}), strict=False) except bv.ValidationError as e: prefix = 'f.i: ' self.assertEqual(prefix, str(e)[:len(prefix)]) raise # Test that validation error references outer union and inner structs with self.assertRaises(bv.ValidationError): try: json_decode(bv.Union(U), json.dumps({'t': {'f': {'i': {}}}}), strict=False, old_style=True) except bv.ValidationError as e: prefix = 't.f.i: ' self.assertEqual(prefix, str(e)[:len(prefix)]) raise def test_type_name_with_module(self): class Foo(): def __init__(self): pass assert bv.type_name_with_module(Foo) == "test.test_python_gen.Foo" assert bv.type_name_with_module(int) == "builtins.int" if six.PY3 else "__builtin__.int" test_spec = """\ namespace ns import ns2 struct A "Sample struct doc." a String "Sample field doc." b Int64 struct B extends A c Bytes struct C extends B d Float64 struct D a String b UInt64 = 10 c String? d List(Int64?) e Map(String, String?) struct E a String = "test" b UInt64 = 10 c Int64? struct DocTest b Boolean "If :val:`true` then..." t String "References :type:`D`." union_closed U "Sample union doc." t0 "Sample field doc." t1 String t2 union UOpen extends U t3 union_closed UExtend extends U t3 union_closed UExtend2 extends U t3 union_closed UExtendExtend extends UExtend t4 union V t0 t1 String t2 String? t3 S t4 S? t5 U t6 U? t7 Resource t8 Resource? t9 List(String) t10 List(U) t11 Map(String, Int32) t12 Map(String, U) struct S f String struct T f String struct SExtendEmpty extends S "Subclass with no fields added" struct SExtend extends S g String struct SExtend2 extends S g String struct Resource union_closed file File folder Folder name String struct File extends Resource size UInt64 struct Folder extends Resource "Regular folder" # Differs from Resource because it's a catch-all struct ResourceLax union file File2 folder Folder2 name String struct File2 extends ResourceLax size UInt64 struct Folder2 extends ResourceLax "Regular folder" struct ImportTestS extends ns2.BaseS a String union_closed ImportTestU extends ns2.BaseU a UInt64 union_closed U2 a b OptionalS struct OptionalS f1 String = "hello" f2 UInt64 = 3 struct S2 f1 OptionalS alias AliasedS2 = S2 alias AliasedString = String(max_length=10) struct ContainsAlias s AliasedString struct S3 u ns2.BaseU = z """ test_ns2_spec = """\ namespace ns2 struct BaseS "This is a test \u2650" z Int64 union_closed BaseU z x String alias AliasedBaseU = BaseU """ class TestGeneratedPython(unittest.TestCase): def setUp(self): # Sanity check: stone must be importable for the compiler to work importlib.import_module('stone') # Compile spec by calling out to stone p = subprocess.Popen( [sys.executable, '-m', 'stone.cli', 'python_types', 'output', '-', '--', '--package', 'output'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = p.communicate( input=(test_spec + test_ns2_spec).encode('utf-8')) if p.wait() != 0: raise AssertionError('Could not execute stone tool: %s' % stderr.decode('utf-8')) self.ns2 = importlib.import_module('output.ns2') self.ns = importlib.import_module('output.ns') self.encode = ss.json_encode self.compat_obj_encode = ss.json_compat_obj_encode self.decode = ss.json_decode self.compat_obj_decode = ss.json_compat_obj_decode def test_docstring(self): # Check that the docstrings from the spec have in some form made it # into the Python docstrings for the generated objects. self.assertIn('Sample struct doc.', self.ns.A.__doc__) self.assertIn('Sample field doc.', self.ns.A.__doc__) self.assertIn('Sample union doc.', self.ns.U.__doc__) self.assertIn('Sample field doc.', self.ns.U.t0.__doc__) # Test doc conversion of Python bool. self.assertIn('``True``', self.ns.DocTest.__doc__) # Test doc converts type reference to sphinx-friendly representation. self.assertIn(':class:`D`', self.ns.DocTest.__doc__) def test_aliases(self): # The left is a validator, the right is the struct devs can use... self.assertEqual(self.ns.AliasedS2, self.ns.S2) def test_struct_decoding(self): d = self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'b': 1, 'c': 'C', 'd': [], 'e': {}})) self.assertIsInstance(d, self.ns.D) self.assertEqual(d.a, 'A') self.assertEqual(d.b, 1) self.assertEqual(d.c, 'C') self.assertEqual(d.d, []) self.assertEqual(d.e, {}) # Test with missing value for nullable field d = self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'b': 1, 'd': [], 'e': {}})) self.assertEqual(d.a, 'A') self.assertEqual(d.b, 1) self.assertEqual(d.c, None) self.assertEqual(d.d, []) self.assertEqual(d.e, {}) # Test with missing value for field with default d = self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'c': 'C', 'd': [1], 'e': {'one': 'two'}})) self.assertEqual(d.a, 'A') self.assertEqual(d.b, 10) self.assertEqual(d.c, 'C') self.assertEqual(d.d, [1]) self.assertEqual(d.e, {'one': 'two'}) # Test with explicitly null value for nullable field d = self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'c': None, 'd': [1, 2], 'e': {'one': 'two'}})) self.assertEqual(d.a, 'A') self.assertEqual(d.c, None) self.assertEqual(d.d, [1, 2]) self.assertEqual(d.e, {'one': 'two'}) # Test with None and numbers in list d = self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'c': None, 'd': [None, 1], 'e': {'one': None, 'two': 'three'}})) self.assertEqual(d.a, 'A') self.assertEqual(d.c, None) self.assertEqual(d.d, [None, 1]) self.assertEqual(d.e, {'one': None, 'two': 'three'}) # Test with explicitly null value for field with default with self.assertRaises(bv.ValidationError) as cm: self.decode(bv.Struct(self.ns.D), json.dumps({'a': 'A', 'b': None})) self.assertEqual("b: expected integer, got null", str(cm.exception)) def test_union_decoding_old(self): v = self.decode(bv.Union(self.ns.V), json.dumps('t0')) self.assertIsInstance(v, self.ns.V) # Test verbose representation of a void union member v = self.decode(bv.Union(self.ns.V), json.dumps({'t0': None}), old_style=True) self.assertIsInstance(v, self.ns.V) # Test bad value for void union member with self.assertRaises(bv.ValidationError) as cm: self.decode(bv.Union(self.ns.V), json.dumps({'t0': 10}), old_style=True) self.assertEqual("expected null, got integer", str(cm.exception)) # Test compact representation of a nullable union member with missing value v = self.decode(bv.Union(self.ns.V), json.dumps('t2')) self.assertIsInstance(v, self.ns.V) # Test verbose representation of a nullable union member with missing value v = self.decode(bv.Union(self.ns.V), json.dumps({'t2': None}), old_style=True) self.assertIsInstance(v, self.ns.V) # Test verbose representation of a nullable union member with bad value with self.assertRaises(bv.ValidationError) as cm: self.decode(bv.Union(self.ns.V), json.dumps({'t2': 123}), old_style=True) self.assertEqual("'123' expected to be a string, got integer", str(cm.exception)) def test_union_decoding(self): v = self.decode(bv.Union(self.ns.V), json.dumps('t0')) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t0()) # Test verbose representation of a void union member v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't0'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t0()) # Test extra verbose representation of a void union member v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't0', 't0': None})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t0()) # Test error: extra key with self.assertRaises(bv.ValidationError) as cm: v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't0', 'unk': 123})) self.assertEqual("unexpected key 'unk'", str(cm.exception)) # Test error: bad type with self.assertRaises(bv.ValidationError) as cm: v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 123})) self.assertEqual('tag must be string, got integer', str(cm.exception)) # Test primitive union member v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't1', 't1': 'hello'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t1()) self.assertEqual(v.get_t1(), 'hello') # Test catch-all v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 'z', 'z': 'hello'}), strict=False) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_other()) # Test catch-all is reject if strict with self.assertRaises(bv.ValidationError): self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 'z'})) # Test explicitly using catch-all is reject if strict with self.assertRaises(bv.ValidationError): self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 'other'})) # Test nullable primitive union member with null value v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't2', 't2': None})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t2()) self.assertEqual(v.get_t2(), None) # Test nullable primitive union member that is missing v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't2'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t2()) self.assertEqual(v.get_t2(), None) # Test error: extra key with self.assertRaises(bv.ValidationError) as cm: self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't2', 't2': None, 'unk': 123})) self.assertEqual("unexpected key 'unk'", str(cm.exception)) # Test composite union member v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't3', 'f': 'hello'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t3()) self.assertIsInstance(v.get_t3(), self.ns.S) self.assertEqual(v.get_t3().f, 'hello') # Test error: extra key with self.assertRaises(bv.ValidationError) as cm: self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't3', 'f': 'hello', 'g': 'blah'})) self.assertEqual("t3: unknown field 'g'", str(cm.exception)) # Test composite union member with unknown key, but strict is False v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't3', 'f': 'hello', 'g': 'blah'}), strict=False) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t3()) self.assertIsInstance(v.get_t3(), self.ns.S) self.assertEqual(v.get_t3().f, 'hello') # Test nullable composite union member v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't4', 'f': 'hello'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t4()) self.assertIsInstance(v.get_t4(), self.ns.S) self.assertEqual(v.get_t4().f, 'hello') # Test nullable composite union member that's null v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't4'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t4()) self.assertIsNone(v.get_t4()) # Test stacked unions v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't5', 't5': {'.tag': 't1', 't1': 'hello'}})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t5(), None) self.assertIsInstance(v.get_t5(), self.ns.U) u = v.get_t5() self.assertEqual(u.get_t1(), 'hello') # Test stacked unions with null v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't6'})) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t6(), None) self.assertEqual(v.get_t6(), None) # Test member that enumerates subtypes v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't7', 't7': {'.tag': 'file', 'name': 'test', 'size': 1024}})) self.assertIsInstance(v, self.ns.V) self.assertIsInstance(v.get_t7(), self.ns.File) f = v.get_t7() self.assertEqual(f.name, 'test') self.assertEqual(f.size, 1024) # Test member that enumerates subtypes with null v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't8'})) self.assertIsInstance(v, self.ns.V) self.assertEqual(v.get_t8(), None) # Test member that is a list v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't9', 't9': ['a', 'b']})) self.assertIsInstance(v, self.ns.V) self.assertEqual(v.get_t9(), ['a', 'b']) # Test member that is a list of composites v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't10', 't10': [{'.tag': 't1', 't1': 'hello'}]})) self.assertIsInstance(v, self.ns.V) t10 = v.get_t10() self.assertEqual(t10[0].get_t1(), 'hello') # Test member that is a list of composites (old style) v = self.decode( bv.Union(self.ns.V), json.dumps({'t10': [{'t1': 'hello'}]}), old_style=True) self.assertIsInstance(v, self.ns.V) t10 = v.get_t10() self.assertEqual(t10[0].get_t1(), 'hello') # Test member that has evolved from void to type in non-strict mode. v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't0', 't0': "hello"}), strict=False) self.assertIsInstance(v, self.ns.V) self.assertTrue(v.is_t0()) # test maps v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't11', 't11': {'a': 100}})) self.assertIsInstance(v, self.ns.V) self.assertEqual(v.get_t11(), {'a': 100}) # Test map of composites: # Test member that is a list of composites v = self.decode( bv.Union(self.ns.V), json.dumps({'.tag': 't12', 't12': {'key': {'.tag': 't1', 't1': 'hello'}}})) self.assertIsInstance(v, self.ns.V) t12 = v.get_t12() self.assertEqual(t12['key'].get_t1(), 'hello') # Test member that is a list of composites (old style) v = self.decode( bv.Union(self.ns.V), json.dumps({'t12': {'another key': {'t1': 'hello again'}}}), old_style=True) self.assertIsInstance(v, self.ns.V) t12 = v.get_t12() self.assertEqual(t12['another key'].get_t1(), 'hello again') def test_union_decoding_with_optional_struct(self): # Simulate that U2 used to have a field b with no value, but it's since # been evolved to a field with an optional struct (only has optional # fields). u2 = self.decode( bv.Union(self.ns.U2), json.dumps({'.tag': 'b'})) self.assertIsInstance(u2, self.ns.U2) b = u2.get_b() self.assertIsInstance(b, self.ns.OptionalS) self.assertEqual(b.f1, 'hello') self.assertEqual(b.f2, 3) def test_union_equality_with_object(self): """Should not throw an error when comparing with object. Object is a superclass of Union, but it should not be considered for equality. """ u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) self.assertFalse(u == object()) def test_union_equality_with_tag(self): u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) u_equal = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) u_unequal = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't2'})) self.assertEqual(u, u_equal) self.assertEqual(hash(u), hash(u_equal)) self.assertNotEqual(u, u_unequal) self.assertNotEqual(hash(u), hash(u_unequal)) def test_union_equality_with_value(self): u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't1', 't1': 'a'})) u_equal = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't1', 't1': 'a'})) u_unequal = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't1', 't1': 'b'})) self.assertEqual(u, u_equal) self.assertEqual(hash(u), hash(u_equal)) self.assertNotEqual(u, u_unequal) self.assertNotEqual(hash(u), hash(u_unequal)) def test_union_equality_with_closed_and_open(self): """A closed union should be considered equal to an open union if they have a direct inheritance relationship.""" u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) u_open = self.decode(bv.Union(self.ns.UOpen), json.dumps({'.tag': 't0'})) self.assertEqual(u, u_open) self.assertEqual(hash(u), hash(u_open)) def test_union_equality_with_different_types(self): """Unions of different types that do not have an inheritance relationship are not considered equal to each other.""" u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) v = self.decode(bv.Union(self.ns.V), json.dumps({'.tag': 't0'})) self.assertNotEqual(u, v) # They still hash to the same value, since they have the same tag and value, but this is # fine since we don't expect to use a large number of unions as dict keys. self.assertEqual(hash(u), hash(v)) # U_extend and U_extend2 are indirectly related because they both extend U, but they do not # have a direct line of inheritance to each other. u_extend = self.decode(bv.Union(self.ns.UExtend), json.dumps({'.tag': 't0'})) u_extend2 = self.decode(bv.Union(self.ns.UExtend2), json.dumps({'.tag': 't0'})) self.assertNotEqual(u_extend, u_extend2) # They still hash to the same value, since they have the same tag and value, but this is # fine since we don't expect to use a large number of unions as dict keys. self.assertEqual(hash(u_extend), hash(u_extend2)) def test_extended_union_equality(self): """Unions which subclass each other are considered equal to each other.""" u = self.decode(bv.Union(self.ns.U), json.dumps({'.tag': 't0'})) u_extend = self.decode(bv.Union(self.ns.UExtend), json.dumps({'.tag': 't0'})) u_extend_extend = self.decode(bv.Union(self.ns.UExtendExtend), json.dumps({'.tag': 't0'})) self.assertEqual(u, u_extend) self.assertEqual(hash(u), hash(u_extend)) self.assertEqual(u, u_extend_extend) self.assertEqual(hash(u), hash(u_extend_extend)) self.assertEqual(u_extend, u_extend_extend) self.assertEqual(hash(u_extend), hash(u_extend_extend)) def test_struct_decoding_with_optional_struct(self): opt_s = self.decode( bv.Struct(self.ns.OptionalS), json.dumps(None)) self.assertEqual(opt_s.f1, 'hello') self.assertEqual(opt_s.f2, 3) # Simulate that S2 used to have no fields, but now it has a new field # that is an optional struct (only has optional fields). s2 = self.decode( bv.Struct(self.ns.S2), json.dumps({})) self.assertIsInstance(s2, self.ns.S2) self.assertIsInstance(s2.f1, self.ns.OptionalS) self.assertEqual(s2.f1.f1, 'hello') self.assertEqual(s2.f1.f2, 3) def test_struct_equality_with_object(self): """Should not throw an error when comparing with object. Object is a superclass of Struct, but it should not be considered for equality. """ s = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'F'})) self.assertFalse(s == object()) def test_struct_equality_with_value(self): s = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'F'})) s_equal = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'F'})) s_unequal = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'G'})) self.assertEqual(s, s_equal) self.assertNotEqual(s, s_unequal) def test_struct_equality_with_different_types(self): """Structs of different types that do not have an inheritance relationship are not considered equal to each other.""" s = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'F'})) t = self.decode(bv.Struct(self.ns.T), json.dumps({'f': 'F'})) self.assertNotEqual(s, t) # S_extend and S_extend2 are indirectly related because they both extend S, but they do not # have a direct line of inheritance to each other. s_extend = self.decode(bv.Struct(self.ns.SExtend), json.dumps({'f': 'F', 'g': 'G'})) s_extend2 = self.decode(bv.Struct(self.ns.SExtend2), json.dumps({'f': 'F', 'g': 'G'})) self.assertNotEqual(s_extend, s_extend2) def test_extended_struct_equality(self): """Structs which subclass each other are considered equal to each other if they have the exact same fields.""" s = self.decode(bv.Struct(self.ns.S), json.dumps({'f': 'F'})) s_extend_empty = self.decode(bv.Struct(self.ns.SExtendEmpty), json.dumps({'f': 'F'})) s_extend = self.decode(bv.Struct(self.ns.SExtend), json.dumps({'f': 'F', 'g': 'G'})) self.assertEqual(s, s_extend_empty) self.assertNotEqual(s, s_extend) def test_union_encoding(self): # Test void union member v_t0 = self.ns.V.t0 self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t0), {'.tag': 't0'}) # Test that the .tag key comes first v = self.compat_obj_encode(bv.Union(self.ns.V), v_t0) self.assertEqual(list(v.keys())[0], '.tag') # Test primitive union member v_t1 = self.ns.V.t1('hello') self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t1), {'.tag': 't1', 't1': 'hello'}) # Test nullable primitive union member v_t2 = self.ns.V.t2('hello') self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t2), {'.tag': 't2', 't2': 'hello'}) # Test nullable primitive union member that's null v_t2 = self.ns.V.t2(None) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t2), {'.tag': 't2'}) # Test composite union member s = self.ns.S(f='hello') v_t3 = self.ns.V.t3(s) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t3), {'.tag': 't3', 'f': 'hello'}) # Test nullable composite union member s = self.ns.S(f='hello') v_t4 = self.ns.V.t4(s) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t4), {'.tag': 't4', 'f': 'hello'}) # Test nullable composite union member that's null v_t4 = self.ns.V.t4(None) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t4), {'.tag': 't4'}) # Test stacked unions v_t5 = self.ns.V.t5(self.ns.U.t1('hello')) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t5), {'.tag': 't5', 't5': {'.tag': 't1', 't1': 'hello'}}) # Test stacked unions with null v_t6 = self.ns.V.t6(None) self.assertEqual(self.compat_obj_encode(bv.Union(self.ns.V), v_t6), {'.tag': 't6'}) # Test member that enumerates subtypes v_t7 = self.ns.V.t7(self.ns.File(name='test', size=1024)) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns.V), v_t7), {'.tag': 't7', 't7': {'.tag': 'file', 'name': 'test', 'size': 1024}}) # Test member that enumerates subtypes but is null v_t8 = self.ns.V.t8(None) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns.V), v_t8), {'.tag': 't8'}) # Test member that is a list v_t9 = self.ns.V.t9(['a', 'b']) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns.V), v_t9), {'.tag': 't9', 't9': ['a', 'b']}) # Test member that is a map v_t11 = self.ns.V.t11({'a_key': 404}) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns.V), v_t11), {'.tag': 't11', 't11': {'a_key': 404}} ) def test_list_coding(self): # Test decoding list of composites v = self.decode( bv.List(bv.Struct(self.ns.S)), json.dumps([{'f': 'Test'}])) self.assertIsInstance(v, list) self.assertIsInstance(v[0], self.ns.S) self.assertEqual(v[0].f, 'Test') # Test encoding list of composites v = self.encode( bv.List(bv.Struct(self.ns.S)), [self.ns.S('Test')]) self.assertEqual(v, json.dumps([{'f': 'Test'}])) def test_objs(self): # Test initializing struct params (also tests parent class fields) a = self.ns.C(a='test', b=123, c=b'\x00', d=3.14) self.assertEqual(a.a, 'test') self.assertEqual(a.b, 123) self.assertEqual(a.c, b'\x00') self.assertEqual(a.d, 3.14) # Test that void union member is available as a class attribute self.assertIsInstance(self.ns.U.t0, self.ns.U) # Test that non-void union member is callable (should be a method) self.assertTrue(callable(self.ns.U.t1)) def test_struct_enumerated_subtypes_encoding(self): # Test serializing a leaf struct from the root struct fi = self.ns.File(name='test.doc', size=100) self.assertEqual( self.compat_obj_encode(bv.StructTree(self.ns.Resource), fi), {'.tag': 'file', 'name': 'test.doc', 'size': 100}) # Test that the .tag key comes first v = self.compat_obj_encode(bv.StructTree(self.ns.Resource), fi) self.assertEqual(list(v.keys())[0], '.tag') # Test serializing a leaf struct as the base and target self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns.File), fi), {'name': 'test.doc', 'size': 100}) def test_struct_enumerated_subtypes_decoding(self): # Test deserializing a leaf struct from the root struct fi = self.compat_obj_decode( bv.StructTree(self.ns.Resource), {'.tag': 'file', 'name': 'test.doc', 'size': 100}) self.assertIsInstance(fi, self.ns.File) self.assertEqual(fi.name, 'test.doc') self.assertEqual(fi.size, 100) # Test deserializing leaf struct with unknown type tag with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( bv.StructTree(self.ns.Resource), {'.tag': 'unk', 'name': 'test.doc'}) self.assertEqual("unknown subtype 'unk'", str(cm.exception)) # Test deserializing leaf struct with bad JSON type for type tag with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( bv.StructTree(self.ns.Resource), {'.tag': 123, 'name': 'test.doc'}) self.assertEqual(".tag: expected string, got integer", str(cm.exception)) # Test deserializing an unknown leaf in strict mode with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( bv.StructTree(self.ns.Resource), {'.tag': 'symlink', 'name': 'test'}) self.assertEqual("unknown subtype 'symlink'", str(cm.exception)) # Test deserializing an unknown leaf in non-strict mode r = self.compat_obj_decode( bv.StructTree(self.ns.ResourceLax), {'.tag': 'symlink', 'name': 'test'}, strict=False) self.assertIsInstance(r, self.ns.ResourceLax) self.assertEqual(r.name, 'test') # Test deserializing an unknown leaf in non-strict mode, but with no # catch-all with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( bv.StructTree(self.ns.Resource), {'.tag': 'symlink', 'name': 'test'}, strict=False) self.assertEqual( "unknown subtype 'symlink' and 'Resource' is not a catch-all", str(cm.exception)) def test_defaults(self): # Test void type v = bv.Void() self.assertTrue(v.has_default()) self.assertEqual(v.get_default(), None) # Test nullable type n = bv.Nullable(bv.Struct(self.ns.D)) self.assertTrue(n.has_default()) self.assertEqual(n.get_default(), None) # Test struct where all fields have defaults s = bv.Struct(self.ns.E) self.assertTrue(s.has_default()) s.get_default() # Test struct where not all fields have defaults s = bv.Struct(self.ns.D) self.assertFalse(s.has_default()) self.assertRaises(AssertionError, s.get_default) def tearDown(self): # Clear output of stone tool after all tests. shutil.rmtree('output') def test_msgpack(self): # Do a limited amount of testing just to make sure that unicode # handling and byte array handling are functional. # If the machine doesn't have msgpack, don't worry about these tests. try: from stone_serializers import ( msgpack_encode, msgpack_decode, ) except ImportError: return b = self.ns.B(a='hi', b=32, c=b'\x00\x01') s = msgpack_encode(bv.Struct(self.ns.B), b) b2 = msgpack_decode(bv.Struct(self.ns.B), s) self.assertEqual(b.a, b2.a) self.assertEqual(b.b, b2.b) self.assertEqual(b.c, b2.c) bs = b'\x00\x01' s = msgpack_encode(bv.Bytes(), bs) bs2 = msgpack_decode(bv.Bytes(), s) self.assertEqual(bs, bs2) u = '\u2650' s = msgpack_encode(bv.String(), u) u2 = msgpack_decode(bv.String(), s) self.assertEqual(u, u2) def test_alias_validators(self): def aliased_string_validator(val): if ' ' in val: raise bv.ValidationError('No spaces allowed') aliased_validators = { self.ns.AliasedString_validator: aliased_string_validator} # # Test decoding # with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( self.ns.AliasedString_validator, 'hi there', alias_validators=aliased_validators) self.assertEqual("No spaces allowed", str(cm.exception)) with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_decode( bv.Struct(self.ns.ContainsAlias), {'s': 'hi there'}, alias_validators=aliased_validators) self.assertEqual("s: No spaces allowed", str(cm.exception)) # # Test encoding # with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode( self.ns.AliasedString_validator, 'hi there', alias_validators=aliased_validators) self.assertEqual("No spaces allowed", str(cm.exception)) ca = self.ns.ContainsAlias(s='hi there') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode( bv.Struct(self.ns.ContainsAlias), ca, alias_validators=aliased_validators) self.assertEqual("s: No spaces allowed", str(cm.exception)) def test_struct_union_default(self): s = self.ns.S3() assert s.u == self.ns2.BaseU.z # Adapted from: # http://code.activestate.com/recipes/306860-proleptic-gregorian-dates-and-strftime-before-1900/ # Make sure that the day names are in order from 0001/01/01 until # 2000/08/01 class TestCustomStrftime(unittest.TestCase): def test_strftime(self): s = stone_strftime(datetime.date(1800, 9, 23), '%Y has the same days as 1980 and 2008') assert s == '1800 has the same days as 1980 and 2008' # Get the weekdays. Can't hard code them; they could be localized. days = [] for i in range(1, 10): days.append(datetime.date(2000, 1, i).strftime('%A')) nextday = {} for i in range(8): nextday[days[i]] = days[i + 1] startdate = datetime.date(1, 1, 1) enddate = datetime.date(2000, 8, 1) prevday = stone_strftime(startdate, '%A') one_day = datetime.timedelta(1) testdate = startdate + one_day while testdate < enddate: day = stone_strftime(testdate, '%A') assert nextday[prevday] == day, str(testdate) prevday = day testdate = testdate + one_day class CallerPermissionsTest(CallerPermissionsInterface): def __init__(self, permissions): self._permissions = permissions @property def permissions(self): return self._permissions test_tagged_spec = """\ namespace ns3 struct A "Sample struct doc." a String "Sample field doc." struct B extends A h String union_closed U "Sample union doc." t0 "Sample field doc." union UOpen extends U t4 struct Resource union file File name String struct File extends Resource size UInt64 """ test_tagged_patched_spec = """\ namespace ns3 import ns4 alias TestAlias = String @ns4.TestFullHashRedactor patch struct A b Float64 @ns4.TestFullHashRedactor @ns4.Deprecated @ns4.InternalOnly "A patched, internal-only field." c String? @ns4.Preview @ns4.InternalOnly @ns4.TestFullBlotRedactor d List(X) @ns4.InternalOnly e Map(String, String?) @ns4.TestFullBlotRedactor @ns4.InternalOnly f X @ns4.InternalOnly g Int64 @ns4.TestFullBlotRedactor @ns4.AlphaOnly struct X a String @ns4.TestPartialBlotRedactor b String @ns4.InternalOnly @ns4.TestPartialHashRedactor patch struct B x String @ns4.InternalOnly y String @ns4.AlphaOnly @ns4.TestFullHashRedactor patch union_closed U t1 String @ns4.TestPartialHashRedactor @ns4.InternalOnly t2 List(X) @ns4.AlphaOnly t3 List(X) t_void @ns4.TestVoidField patch union UOpen t5 TestAlias @ns4.InternalOnly t6 String @ns4.AlphaOnly patch struct Resource x X @ns4.InternalOnly patch struct File y String @ns4.TestPartialHashRedactor @ns4.InternalOnly struct S2 a List(String) @ns4.TestFullBlotRedactor b Map(String, String) @ns4.TestFullHashRedactor union U2 t1 List(String) @ns4.TestFullHashRedactor t2 Map(String, String) @ns4.TestFullBlotRedactor """ test_tagged_spec_2 = """\ namespace ns4 annotation InternalOnly = Omitted("internal") annotation AlphaOnly = Omitted("alpha") annotation TestVoidField = Omitted("test_void_field") annotation Deprecated = Deprecated() annotation Preview = Preview() annotation TestFullHashRedactor = RedactedHash() annotation TestFullBlotRedactor = RedactedBlot() annotation TestPartialHashRedactor = RedactedHash("()(\\-hash\\-)()") annotation TestPartialBlotRedactor = RedactedBlot("()(\\-blot\\-)()") """ class TestAnnotationsGeneratedPython(unittest.TestCase): def setUp(self): # Sanity check: stone must be importable for the compiler to work importlib.import_module('stone') # Compile spec by calling out to stone p = subprocess.Popen( [sys.executable, '-m', 'stone.cli', 'python_types', 'output', '-', '--', '--package', 'output'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = p.communicate( input=(test_tagged_spec + test_tagged_patched_spec + test_tagged_spec_2).encode('utf-8')) if p.wait() != 0: raise AssertionError('Could not execute stone tool: %s' % stderr.decode('utf-8')) self.ns4 = importlib.import_module('output.ns4') self.ns3 = importlib.import_module('output.ns3') self.encode = ss.json_encode self.compat_obj_encode = ss.json_compat_obj_encode self.decode = ss.json_decode self.compat_obj_decode = ss.json_compat_obj_decode self.default_cp = CallerPermissionsTest([]) self.internal_cp = CallerPermissionsTest(['internal']) self.alpha_cp = CallerPermissionsTest(['alpha']) self.internal_and_alpha_cp = CallerPermissionsTest(['internal', 'alpha']) def test_struct_parent_decoding(self): json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'g': 4, } # test full super-type a = self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(a, self.ns3.A) self.assertEqual(a.a, 'A') self.assertEqual(a.b, 1) self.assertEqual(a.c, 'C') self.assertEqual(a.d[0].a, 'A') self.assertEqual(a.d[0].b, 'B') self.assertEqual(a.e, {}) self.assertEqual(a.f.a, 'A') self.assertEqual(a.f.b, 'B') self.assertEqual(a.g, 4) json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, } # test internal-only type a = self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIsInstance(a, self.ns3.A) self.assertEqual(a.a, 'A') self.assertEqual(a.b, 1) self.assertEqual(a.c, 'C') self.assertEqual(a.d[0].a, 'A') self.assertEqual(a.d[0].b, 'B') self.assertEqual(a.e, {}) self.assertEqual(a.f.a, 'A') self.assertEqual(a.f.b, 'B') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.g json_data = { 'a': 'A', 'g': 4, } # test alpha-only type a = self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertIsInstance(a, self.ns3.A) self.assertEqual(a.a, 'A') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.b # optional field, so doesn't raise error self.assertEqual(a.c, None) with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.d with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.e with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.f self.assertEqual(a.g, 4) json_data = { 'a': 'A', } # test external-only type a = self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIsInstance(a, self.ns3.A) self.assertEqual(a.a, 'A') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.b # optional field, so doesn't raise error self.assertEqual(a.c, None) with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.d with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.e with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.f with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.g json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'g': 4, } # test internal-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("unknown field 'g'", str(cm.exception)) # test alpha-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIn("unknown field", str(cm.exception)) # test external-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIn("unknown field", str(cm.exception)) json_data = { 'a': 'A', 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, } # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("missing required field 'b'", str(cm.exception)) json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, } # test missing nested required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("d: missing required field 'b'", str(cm.exception)) json_data = { 'a': 'A', 'b': 1, 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, } # test missing optional internal field for internal caller a = self.decode( bv.Struct(self.ns3.A), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIsInstance(a, self.ns3.A) self.assertEqual(a.a, 'A') self.assertEqual(a.b, 1) # optional field, so doesn't raise error self.assertEqual(a.c, None) self.assertEqual(a.d[0].a, 'A') self.assertEqual(a.d[0].b, 'B') self.assertEqual(a.e, {}) self.assertEqual(a.f.a, 'A') self.assertEqual(a.f.b, 'B') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement a.g def test_struct_child_decoding(self): json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'g': 4, 'h': 'H', 'x': 'X', 'y': 'Y', } # test full super-type b = self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(b, self.ns3.B) self.assertEqual(b.a, 'A') self.assertEqual(b.b, 1) self.assertEqual(b.c, 'C') self.assertEqual(b.d[0].a, 'A') self.assertEqual(b.d[0].b, 'B') self.assertEqual(b.e, {}) self.assertEqual(b.f.a, 'A') self.assertEqual(b.f.b, 'B') self.assertEqual(b.g, 4) self.assertEqual(b.h, 'H') self.assertEqual(b.x, 'X') self.assertEqual(b.y, 'Y') json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'h': 'H', 'x': 'X', } # test internal-only type b = self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIsInstance(b, self.ns3.B) self.assertEqual(b.a, 'A') self.assertEqual(b.b, 1) self.assertEqual(b.c, 'C') self.assertEqual(b.d[0].a, 'A') self.assertEqual(b.d[0].b, 'B') self.assertEqual(b.e, {}) self.assertEqual(b.f.a, 'A') self.assertEqual(b.f.b, 'B') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.g self.assertEqual(b.h, 'H') self.assertEqual(b.x, 'X') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.y json_data = { 'a': 'A', 'g': 4, 'h': 'H', 'y': 'Y', } # test alpha-only type b = self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertIsInstance(b, self.ns3.B) self.assertEqual(b.a, 'A') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.b # optional field, so doesn't raise error self.assertEqual(b.c, None) with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.d with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.e with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.f self.assertEqual(b.g, 4) self.assertEqual(b.h, 'H') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.x self.assertEqual(b.y, 'Y') json_data = { 'a': 'A', 'h': 'H', } # test external-only type b = self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIsInstance(b, self.ns3.B) self.assertEqual(b.a, 'A') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.b # optional field, so doesn't raise error self.assertEqual(b.c, None) with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.d with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.e with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.f with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.g self.assertEqual(b.h, 'H') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.x with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.y json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'g': 4, 'h': 'H', 'x': 'X', 'y': 'Y', } # test internal-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIn("unknown field", str(cm.exception)) # test alpha-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIn("unknown field", str(cm.exception)) # test external-only type raises with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIn("unknown field", str(cm.exception)) json_data = { 'a': 'A', 'c': 'C', 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'h': 'H', 'x': 'X', } # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("missing required field 'b'", str(cm.exception)) json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [ { 'a': 'A', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'h': 'H', 'x': 'X', } # test missing nested required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("d: missing required field 'b'", str(cm.exception)) json_data = { 'a': 'A', 'b': 1, 'd': [ { 'a': 'A', 'b': 'B', }, ], 'e': {}, 'f': { 'a': 'A', 'b': 'B', }, 'h': 'H', 'x': 'X', } # test missing optional internal field for internal caller b = self.decode( bv.Struct(self.ns3.B), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertIsInstance(b, self.ns3.B) self.assertEqual(b.a, 'A') self.assertEqual(b.b, 1) # optional field, so doesn't raise error self.assertEqual(b.c, None) self.assertEqual(b.d[0].a, 'A') self.assertEqual(b.d[0].b, 'B') self.assertEqual(b.e, {}) self.assertEqual(b.f.a, 'A') self.assertEqual(b.f.b, 'B') with self.assertRaises(AttributeError): # pylint: disable=pointless-statement b.g self.assertEqual(b.h, 'H') self.assertEqual(b.x, 'X') def test_union_closed_parent_decoding(self): # test all tags json_data = { '.tag': 't0', } u = self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.U) self.assertTrue(u.is_t0()) json_data = { '.tag': 't1', 't1': 't1_str' } u = self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.U) self.assertTrue(u.is_t1()) self.assertEqual(u.get_t1(), 't1_str') json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } u = self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.U) self.assertTrue(u.is_t2()) self.assertEqual(u.get_t2()[0].a, 'A') self.assertEqual(u.get_t2()[0].b, 'B') json_data = { '.tag': 't1', 't1': 't1_str' } # test internal tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.default_cp) self.assertEqual("unknown tag 't1'", str(cm.exception)) # test internal tag raises for alpha caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertEqual("unknown tag 't1'", str(cm.exception)) json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } # test alpha tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.default_cp) self.assertEqual("unknown tag 't2'", str(cm.exception)) # test alpha tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("unknown tag 't2'", str(cm.exception)) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', }, ] } # test missing required internal field for external caller u = self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIsInstance(u, self.ns3.U) self.assertTrue(u.is_t3()) self.assertEqual(u.get_t3()[0].a, 'A') with self.assertRaises(AttributeError): # pylint: disable=expression-not-assigned u.get_t3()[0].b # test missing required internal field for alpha caller u = self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertIsInstance(u, self.ns3.U) self.assertTrue(u.is_t3()) self.assertEqual(u.get_t3()[0].a, 'A') with self.assertRaises(AttributeError): # pylint: disable=expression-not-assigned u.get_t3()[0].b # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.U), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("t3: missing required field 'b'", str(cm.exception)) def test_union_open_child_decoding(self): # test all tags json_data = { '.tag': 't0', } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t0()) json_data = { '.tag': 't1', 't1': 't1_str' } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t1()) self.assertEqual(u.get_t1(), 't1_str') json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t2()) self.assertEqual(u.get_t2()[0].a, 'A') self.assertEqual(u.get_t2()[0].b, 'B') json_data = { '.tag': 't1', 't1': 't1_str' } # test internal tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.default_cp) self.assertEqual("unknown tag 't1'", str(cm.exception)) # test internal tag raises for alpha caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertEqual("unknown tag 't1'", str(cm.exception)) json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } # test alpha tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.default_cp) self.assertEqual("unknown tag 't2'", str(cm.exception)) # test alpha tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("unknown tag 't2'", str(cm.exception)) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', }, ] } # test missing required internal field for external caller u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t3()) self.assertEqual(u.get_t3()[0].a, 'A') with self.assertRaises(AttributeError): # pylint: disable=expression-not-assigned u.get_t3()[0].b # test missing required internal field for alpha caller u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t3()) self.assertEqual(u.get_t3()[0].a, 'A') with self.assertRaises(AttributeError): # pylint: disable=expression-not-assigned u.get_t3()[0].b # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("t3: missing required field 'b'", str(cm.exception)) json_data = { '.tag': 't4', } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t4()) json_data = { '.tag': 't5', 't5': 't5_str' } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t5()) self.assertEqual(u.get_t5(), 't5_str') json_data = { '.tag': 't6', 't6': 't6_str' } u = self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(u, self.ns3.UOpen) self.assertTrue(u.is_t6()) self.assertEqual(u.get_t6(), 't6_str') # test alpha tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.default_cp) self.assertEqual("unknown tag 't6'", str(cm.exception)) # test alpha tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.Union(self.ns3.UOpen), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("unknown tag 't6'", str(cm.exception)) def test_enumerated_subtype_decoding(self): json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, 'x': { 'a': 'A', 'b': 'B', }, 'y': 'Y', } # test full super-type f = self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.internal_and_alpha_cp) self.assertIsInstance(f, self.ns3.File) self.assertEqual(f.name, 'File1') self.assertEqual(f.size, 5) self.assertEqual(f.x.a, 'A') self.assertEqual(f.x.b, 'B') self.assertEqual(f.y, 'Y') # test raises with interal field for external caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIn("unknown field", str(cm.exception)) json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, 'y': 'Y', } # test raises with missing required interal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("missing required field 'x'", str(cm.exception)) json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, 'x': { 'a': 'A', 'b': 'B', }, } # test raises with missing required interal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("missing required field 'y'", str(cm.exception)) json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, 'x': { 'a': 'A', }, 'y': 'Y', } # test raises with missing required interal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.internal_cp) self.assertEqual("x: missing required field 'b'", str(cm.exception)) json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, } # test external-only type f = self.decode( bv.StructTree(self.ns3.File), json.dumps(json_data), caller_permissions=self.default_cp) self.assertIsInstance(f, self.ns3.File) self.assertEqual(f.name, 'File1') self.assertEqual(f.size, 5) with self.assertRaises(AttributeError): # pylint: disable=pointless-statement f.x with self.assertRaises(AttributeError): # pylint: disable=pointless-statement f.y def test_struct_child_encoding(self): json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [{'a': 'A', 'b': 'B'}], 'e': {}, 'f': {'a': 'A', 'b': 'B'}, 'g': 4, } # test full super-type ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_and_alpha_cp), json_data) # test missing required internal field for internal and alpha caller ai = self.ns3.A( a='A', c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4) with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'b'", str(cm.exception)) # test missing nested required internal field for internal and alpha caller ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='A')], e={}, f=self.ns3.X(a='A', b='B'), g=4) with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("d: missing required field 'b'", str(cm.exception)) # test missing required alpha field for internal and alpha caller ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B')) with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'g'", str(cm.exception)) json_data = { 'a': 'A', } # test internal and alpha stripped out for external caller ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.default_cp), json_data) json_data = { 'a': 'A', 'g': 4, } # test internal stripped out for alpha caller self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.alpha_cp), json_data) json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [{'a': 'A', 'b': 'B'}], 'e': {}, 'f': {'a': 'A', 'b': 'B'}, } # test alpha stripped out for internal caller self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_cp), json_data) def test_struct_parent_encoding(self): json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [{'a': 'A', 'b': 'B'}], 'e': {}, 'f': {'a': 'A', 'b': 'B'}, 'g': 4, 'h': 'H', 'x': 'X', 'y': 'Y', } # test full super-type bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', x='X', y='Y') self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp), json_data) # test missing required internal field for internal and alpha caller bi = self.ns3.B( a='A', c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', x='X', y='Y') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'b'", str(cm.exception)) # test missing required internal field in child # for internal and alpha caller bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', y='Y') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'x'", str(cm.exception)) # test missing nested required internal field for internal and alpha caller bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', x='X', y='Y') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("d: missing required field 'b'", str(cm.exception)) # test missing required alpha field for internal and alpha caller bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), h='H', x='X', y='Y') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'g'", str(cm.exception)) # test missing required alpha field in child # for internal and alpha caller bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', x='X') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'y'", str(cm.exception)) json_data = { 'a': 'A', 'h': 'H', } # test internal and alpha stripped out for external caller bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='A', b='B')], e={}, f=self.ns3.X(a='A', b='B'), g=4, h='H', x='X', y='Y') self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.default_cp), json_data) json_data = { 'a': 'A', 'g': 4, 'h': 'H', 'y': 'Y', } # test internal stripped out for alpha caller self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.alpha_cp), json_data) json_data = { 'a': 'A', 'b': 1, 'c': 'C', 'd': [{'a': 'A', 'b': 'B'}], 'e': {}, 'f': {'a': 'A', 'b': 'B'}, 'h': 'H', 'x': 'X', } # test alpha stripped out for internal caller self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_cp), json_data) def test_union_closed_parent_encoding(self): # test all tags json_data = { '.tag': 't0', } ui = self.ns3.U.t0 self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't1', 't1': 't1_str' } ui = self.ns3.U.t1('t1_str') self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } ui = self.ns3.U.t2([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', 'b': 'B', }, ], } ui = self.ns3.U.t3([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't1', 't1': 't1_str' } ui = self.ns3.U.t1('t1_str') # test internal tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't1' tag", str(cm.exception)) ui = self.ns3.U.t2([self.ns3.X(a='A', b='B')]) # test alpha tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't2' tag", str(cm.exception)) ui = self.ns3.U.t2([self.ns3.X(a='A', b='B')]) # test alpha tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_cp) self.assertEqual("caller does not have access to 't2' tag", str(cm.exception)) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', }, ] } # test missing required internal field for external caller ui = self.ns3.U.t3([self.ns3.X(a='A')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.default_cp), json_data) # test missing required internal field for alpha caller self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.alpha_cp), json_data) # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_cp) self.assertEqual("t3: missing required field 'b'", str(cm.exception)) def test_union_open_child_encoding(self): # test all tags json_data = { '.tag': 't0', } ui = self.ns3.UOpen.t0 self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't1', 't1': 't1_str' } ui = self.ns3.UOpen.t1('t1_str') self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't2', 't2': [ { 'a': 'A', 'b': 'B', }, ] } ui = self.ns3.UOpen.t2([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', 'b': 'B', }, ], } ui = self.ns3.U.t3([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't4', } ui = self.ns3.UOpen.t4 self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't5', 't5': 't5_str' } ui = self.ns3.UOpen.t5('t5_str') self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't6', 't6': 't6_str' } ui = self.ns3.UOpen.t6('t6_str') self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_and_alpha_cp), json_data) json_data = { '.tag': 't1', 't1': 't1_str' } ui = self.ns3.UOpen.t1('t1_str') # test internal tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't1' tag", str(cm.exception)) ui = self.ns3.UOpen.t2([self.ns3.X(a='A', b='B')]) # test alpha tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't2' tag", str(cm.exception)) ui = self.ns3.UOpen.t2([self.ns3.X(a='A', b='B')]) # test alpha tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_cp) self.assertEqual("caller does not have access to 't2' tag", str(cm.exception)) ui = self.ns3.UOpen.t5('t5_str') # test internal child tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't5' tag", str(cm.exception)) ui = self.ns3.UOpen.t6('t6_str') # test alpha child tag raises for external caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.default_cp) self.assertEqual("caller does not have access to 't6' tag", str(cm.exception)) # test alpha child tag raises for internal caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_cp) self.assertEqual("caller does not have access to 't6' tag", str(cm.exception)) json_data = { '.tag': 't3', 't3': [ { 'a': 'A', }, ] } # test missing required internal field for external caller ui = self.ns3.UOpen.t3([self.ns3.X(a='A')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.default_cp), json_data) # test missing required internal field for alpha caller self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.alpha_cp), json_data) # test missing required internal field for internal caller with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.Union(self.ns3.UOpen), ui, caller_permissions=self.internal_cp) self.assertEqual("t3: missing required field 'b'", str(cm.exception)) def test_enumerated_subtype_encoding(self): json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, 'x': { 'a': 'A', 'b': 'B', }, 'y': 'Y', } # test full super-type fi = self.ns3.File( name='File1', size=5, x=self.ns3.X(a='A', b='B'), y='Y') self.assertEqual( self.compat_obj_encode(bv.StructTree(self.ns3.File), fi, caller_permissions=self.internal_and_alpha_cp), json_data) # test missing required internal parent field for internal and alpha caller fi = self.ns3.File( name='File1', size=5, y='Y') with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.StructTree(self.ns3.File), fi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'x'", str(cm.exception)) # test missing required internal child field for internal and alpha caller fi = self.ns3.File( name='File1', size=5, x=self.ns3.X(a='A', b='B')) with self.assertRaises(bv.ValidationError) as cm: self.compat_obj_encode(bv.StructTree(self.ns3.File), fi, caller_permissions=self.internal_and_alpha_cp) self.assertEqual("missing required field 'y'", str(cm.exception)) json_data = { '.tag': 'file', 'name': 'File1', 'size': 5, } fi = self.ns3.File( name='File1', size=5, x=self.ns3.X(a='A', b='B'), y='Y') # test internal stripped out for external caller self.assertEqual( self.compat_obj_encode(bv.StructTree(self.ns3.File), fi, caller_permissions=self.default_cp), json_data) def test_struct_parent_encoding_with_redaction(self): json_data = { 'a': 'A', 'b': 'e4c2e8edac362acab7123654b9e73432', 'c': '********', 'd': [{'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}], 'e': {'e1': '********'}, 'f': {'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}, 'g': '********', } # test full super-type ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST')], e={'e1': 'e2'}, f=self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST'), g=4) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) json_data = { 'a': 'A', 'b': 'e4c2e8edac362acab7123654b9e73432', 'c': '********', 'd': [{'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}], 'e': {'e1': '********'}, 'f': {'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}, } # test internal type ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST')], e={'e1': 'e2'}, f=self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST'), g=4) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.internal_cp, should_redact=True), json_data) json_data = { 'a': 'A', 'g': '********', } # test alpha type ai = self.ns3.A( a='A', b=1, c='C', d=[self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST')], e={'e1': 'e2'}, f=self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST'), g=4) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.A), ai, caller_permissions=self.alpha_cp, should_redact=True), json_data) def test_struct_child_encoding_with_redaction(self): json_data = { 'a': 'A', 'b': 'e4c2e8edac362acab7123654b9e73432', 'c': '********', 'd': [{'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}], 'e': {'e1': '********'}, 'f': {'a': '***-blot-***', 'b': '3ac5041d7a9d0f27e045f0b15034f186 (***-hash-***)'}, 'g': '********', 'h': 'H', 'x': 'X', 'y': '57cec4137b614c87cb4e24a3d003a3e0', } # test full super-type bi = self.ns3.B( a='A', b=1, c='C', d=[self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST')], e={'e1': 'e2'}, f=self.ns3.X(a='TEST-blot-TEST', b='TEST-hash-TEST'), g=4, h='H', x='X', y='Y') self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.B), bi, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) def test_union_closed_parent_encoding_with_redaction(self): # test all tags json_data = { '.tag': 't0', } ui = self.ns3.U.t0 self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) json_data = { '.tag': 't1', 't1': 'c983987c3b0629b9906c5c7d353409fe' } ui = self.ns3.U.t1('t1_str') self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) json_data = { '.tag': 't2', 't2': [ { 'a': '********', 'b': '9d5ed678fe57bcca610140957afab571', }, ] } ui = self.ns3.U.t2([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) json_data = { '.tag': 't3', 't3': [ { 'a': '********', 'b': '9d5ed678fe57bcca610140957afab571', }, ], } ui = self.ns3.U.t3([self.ns3.X(a='A', b='B')]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) def test_encoding_collections_with_redaction(self): # test that we correctly redact elements in a list/map in a struct json_data = { 'a': ['********'], 'b': { 'key': '74e710825309d622d0b920390ef03edf', } } s = self.ns3.S2(a=['test_str'], b={'key': 'test_str'}) self.assertEqual( self.compat_obj_encode(bv.Struct(self.ns3.S2), s, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) # test that we correctly redact elements in a list/map in a union json_data = { '.tag': 't1', 't1': ['74e710825309d622d0b920390ef03edf'], } ui = self.ns3.U2.t1(['test_str']) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U2), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) json_data = { '.tag': 't2', 't2': {'key': '********'}, } ui = self.ns3.U2.t2({'key': 'test_str'}) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U2), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) def test_encoding_unicode_with_redaction(self): unicode_val = "Unicode val'`~$%&\u53c9\u71d2" json_data = { '.tag': 't2', 't2': [ { 'a': '********', 'b': '89e514e90912003ff11d79560a750510', }, ] } ui = self.ns3.U.t2([self.ns3.X(a=unicode_val, b=unicode_val)]) self.assertEqual( self.compat_obj_encode(bv.Union(self.ns3.U), ui, caller_permissions=self.internal_and_alpha_cp, should_redact=True), json_data) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_python_type_stubs.py000066400000000000000000000400221417406541500210450ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import textwrap MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression import unittest try: # Works for Py 3.3+ from unittest.mock import Mock except ImportError: # See https://github.com/python/mypy/issues/1153#issuecomment-253842414 from mock import Mock # type: ignore from stone.ir import ( Alias, Api, ApiNamespace, Boolean, List, Map, Nullable, String, Struct, StructField, Timestamp, UInt64, Union, UnionField, Void, Float64) from stone.ir.api import ApiRoute from stone.backends.python_type_stubs import PythonTypeStubsBackend ISO_8601_FORMAT = "%Y-%m-%dT%H:%M:%SZ" def _make_backend(): # type: () -> PythonTypeStubsBackend return PythonTypeStubsBackend( target_folder_path=Mock(), args=['--package', 'output'], ) def _make_namespace_with_alias(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_with_alias') struct1 = Struct(name='Struct1', namespace=ns, ast_node=None) struct1.set_attributes(None, [StructField('f1', Boolean(), None, None)]) ns.add_data_type(struct1) alias = Alias(name='AliasToStruct1', namespace=ns, ast_node=None) alias.set_attributes(doc=None, data_type=struct1) ns.add_alias(alias) str_type = String(min_length=3) str_alias = Alias(name='NotUserDefinedAlias', namespace=ns, ast_node=None) str_alias.set_attributes(doc=None, data_type=str_type) ns.add_alias(str_alias) return ns def _make_namespace_with_many_structs(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_with_many_structs') struct1 = Struct(name='Struct1', namespace=ns, ast_node=None) struct1.set_attributes(None, [StructField('f1', Boolean(), None, None)]) ns.add_data_type(struct1) struct2 = Struct(name='Struct2', namespace=ns, ast_node=None) struct2.set_attributes( doc=None, fields=[ StructField('f2', List(UInt64()), None, None), StructField('f3', Timestamp(ISO_8601_FORMAT), None, None), StructField('f4', Map(String(), UInt64()), None, None) ] ) ns.add_data_type(struct2) return ns def _make_namespace_with_nested_types(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_w_nested_types') struct = Struct(name='NestedTypes', namespace=ns, ast_node=None) struct.set_attributes( doc=None, fields=[ StructField( name='NullableList', data_type=Nullable( List(UInt64()) ), doc=None, ast_node=None, ), StructField( name='ListOfNullables', data_type=List( Nullable(UInt64()) ), doc=None, ast_node=None, ) ] ) ns.add_data_type(struct) return ns def _make_namespace_with_a_union(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_with_a_union') u1 = Union(name='Union', namespace=ns, ast_node=None, closed=True) u1.set_attributes( doc=None, fields=[ UnionField( name="first", doc=None, data_type=Void(), ast_node=None ), UnionField( name="last", doc=None, data_type=Void(), ast_node=None ), ], ) ns.add_data_type(u1) # A more interesting case with non-void variants. shape_union = Union(name='Shape', namespace=ns, ast_node=None, closed=True) shape_union.set_attributes( doc=None, fields=[ UnionField( name="point", doc=None, data_type=Void(), ast_node=None ), UnionField( name="circle", doc=None, data_type=Float64(), ast_node=None ), ], ) ns.add_data_type(shape_union) return ns def _make_namespace_with_empty_union(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_with_empty_union') union = Union(name='EmptyUnion', namespace=ns, ast_node=None, closed=True) union.set_attributes( doc=None, fields=[], ) ns.add_data_type(union) return ns def _make_namespace_with_route(): # type: (...) -> ApiNamespace ns = ApiNamespace("_make_namespace_with_route()") mock_ast_node = Mock() route_one = ApiRoute( name="route_one", version=1, ast_node=mock_ast_node, ) route_two = ApiRoute( name="route_one", version=2, ast_node=mock_ast_node, ) ns.add_route(route_one) ns.add_route(route_two) return ns def _make_namespace_with_route_name_conflict(): # type: (...) -> ApiNamespace ns = ApiNamespace("_make_namespace_with_route()") mock_ast_node = Mock() route_one = ApiRoute( name="route_one_v2", version=1, ast_node=mock_ast_node, ) route_two = ApiRoute( name="route_one", version=2, ast_node=mock_ast_node, ) ns.add_route(route_one) ns.add_route(route_two) return ns def _make_namespace_with_nullable_and_dafault_fields(): # type: (...) -> ApiNamespace ns = ApiNamespace('ns_w_nullable__fields') struct = Struct(name='Struct1', namespace=ns, ast_node=None) default_field = StructField( name='DefaultField', data_type=UInt64(), doc=None, ast_node=None, ) default_field.set_default(1) struct.set_attributes( doc=None, fields=[ StructField( name='NullableField', data_type=Nullable( UInt64() ), doc=None, ast_node=None, ), default_field, StructField( name='RequiredField', data_type=UInt64(), doc=None, ast_node=None, ) ] ) ns.add_data_type(struct) return ns def _api(): api = Api(version="1.0") return api _headers = """\ # -*- coding: utf-8 -*- # Auto-generated by Stone, do not modify. # @generated # flake8: noqa # pylint: skip-file {} from stone.backends.python_rsrc import stone_base as bb # type: ignore from stone.backends.python_rsrc import stone_validators as bv # type: ignore T = TypeVar('T', bound=bb.AnnotationType) U = TypeVar('U')""" class TestPythonTypeStubs(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestPythonTypeStubs, self).__init__(*args, **kwargs) self.maxDiff = 1000000 # Increase text diff size def _evaluate_namespace(self, ns): # type: (ApiNamespace) -> typing.Text backend = _make_backend() backend._generate_base_namespace_module(ns) return backend.output_buffer_to_string() def test__generate_base_namespace_module__with_many_structs(self): # type: () -> None ns = _make_namespace_with_many_structs() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class Struct1(bb.Struct): def __init__(self, f1: bool = ...) -> None: ... f1: bb.Attribute[bool] = ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Struct1_validator: bv.Validator = ... class Struct2(bb.Struct): def __init__(self, f2: List[int] = ..., f3: datetime.datetime = ..., f4: Dict[Text, int] = ...) -> None: ... f2: bb.Attribute[List[int]] = ... f3: bb.Attribute[datetime.datetime] = ... f4: bb.Attribute[Dict[Text, int]] = ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Struct2_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, Dict, List, Text, Type, TypeVar, ) import datetime"""))) self.assertEqual(result, expected) def test__generate_base_namespace_module__with_nested_types(self): # type: () -> None ns = _make_namespace_with_nested_types() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class NestedTypes(bb.Struct): def __init__(self, list_of_nullables: List[Optional[int]] = ..., nullable_list: Optional[List[int]] = ...) -> None: ... list_of_nullables: bb.Attribute[List[Optional[int]]] = ... nullable_list: bb.Attribute[Optional[List[int]]] = ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... NestedTypes_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, List, Optional, Text, Type, TypeVar, )"""))) self.assertEqual(result, expected) def test__generate_base_namespace_module_with_union__generates_stuff(self): # type: () -> None ns = _make_namespace_with_a_union() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class Union(bb.Union): first: Union = ... last: Union = ... def is_first(self) -> bool: ... def is_last(self) -> bool: ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Union_validator: bv.Validator = ... class Shape(bb.Union): point: Shape = ... def is_point(self) -> bool: ... def is_circle(self) -> bool: ... @classmethod def circle(cls, val: float) -> Shape: ... def get_circle(self) -> float: ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Shape_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, Text, Type, TypeVar, )"""))) self.assertEqual(result, expected) def test__generate_base_namespace_module_with_empty_union(self): # type: () -> None ns = _make_namespace_with_empty_union() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class EmptyUnion(bb.Union): def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... EmptyUnion_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, Text, Type, TypeVar, )"""))) self.assertEqual(result, expected) def test__generate_routes(self): # type: () -> None ns = _make_namespace_with_route() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} route_one: bb.Route = ... route_one_v2: bb.Route = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( TypeVar, )"""))) self.assertEqual(result, expected) def test__generate_routes_name_conflict(self): # type: () -> None ns = _make_namespace_with_route_name_conflict() with self.assertRaises(RuntimeError) as cm: self._evaluate_namespace(ns) self.assertEqual( 'There is a name conflict between {!r} and {!r}'.format(ns.routes[0], ns.routes[1]), str(cm.exception)) def test__generate_base_namespace_module__with_alias(self): # type: () -> None ns = _make_namespace_with_alias() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class Struct1(bb.Struct): def __init__(self, f1: bool = ...) -> None: ... f1: bb.Attribute[bool] = ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Struct1_validator: bv.Validator = ... AliasToStruct1_validator: bv.Validator = ... AliasToStruct1 = Struct1 NotUserDefinedAlias_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, Text, Type, TypeVar, )"""))) self.assertEqual(result, expected) def test__generate_base_namespace_module__with_nullable_and_default_fields(self): # type: () -> None """ Make sure that only Nullable fields and fields with default values are Optional in mypy """ ns = _make_namespace_with_nullable_and_dafault_fields() result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ {headers} class Struct1(bb.Struct): def __init__(self, required_field: int = ..., nullable_field: Optional[int] = ..., default_field: Optional[int] = ...) -> None: ... required_field: bb.Attribute[int] = ... nullable_field: bb.Attribute[Optional[int]] = ... default_field: bb.Attribute[int] = ... def _process_custom_annotations( self, annotation_type: Type[T], field_path: Text, processor: Callable[[T, U], U], ) -> None: ... Struct1_validator: bv.Validator = ... """).format(headers=_headers.format(textwrap.dedent("""\ from typing import ( Callable, Optional, Text, Type, TypeVar, )"""))) self.assertEqual(expected, result) stone-3.3.1/test/test_python_types.py000066400000000000000000000217601417406541500200200ustar00rootroot00000000000000import textwrap from stone.backends.python_types import PythonTypesBackend from stone.ir import ( AnnotationType, AnnotationTypeParam, Api, ApiNamespace, ApiRoute, CustomAnnotation, Int32, Struct, StructField, Void, ) MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression import unittest class TestGeneratedPythonTypes(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGeneratedPythonTypes, self).__init__(*args, **kwargs) def _mk_route_schema(self): s = Struct('Route', ApiNamespace('stone_cfg'), None) s.set_attributes(None, [], None) return s def _mock_backend(self): # type: () -> PythonTypesBackend return PythonTypesBackend( target_folder_path='output', args=['-r', 'dropbox.dropbox.Dropbox.{ns}_{route}', '--package', 'output']) def _evaluate_namespace(self, ns): # type: (ApiNamespace) -> typing.Text backend = self._mock_backend() route_schema = self._mk_route_schema() backend._generate_routes(route_schema, ns) return backend.output_buffer_to_string() def _evaluate_namespace_definition(self, api, ns): # type: (Api, ApiNamespace) -> typing.Text backend = self._mock_backend() backend._generate_base_namespace_module(api, ns) return backend.output_buffer_to_string() def _evaluate_struct(self, ns, struct): # type: (ApiNamespace, Struct) -> typing.Text backend = self._mock_backend() backend._generate_struct_class(ns, struct) return backend.output_buffer_to_string() def _evaluate_annotation_type(self, ns, annotation_type): # type: (ApiNamespace, AnnotationType) -> typing.Text backend = self._mock_backend() backend._generate_annotation_type_class(ns, annotation_type) return backend.output_buffer_to_string() def test_namespace_comments(self): # type: () -> None ns = ApiNamespace('files') ns.add_doc("Test Doc testing a :type:`Type`.") route_schema = self._mk_route_schema() api = Api('0.0') api.add_route_schema(route_schema) result = self._evaluate_namespace_definition(api, ns) expected = textwrap.dedent("""\ # -*- coding: utf-8 -*- # Auto-generated by Stone, do not modify. # @generated # flake8: noqa # pylint: skip-file \"\"\" Test Doc testing a :class:`Type`. \"\"\" from __future__ import unicode_literals from stone.backends.python_rsrc import stone_base as bb from stone.backends.python_rsrc import stone_validators as bv ROUTES = { } """) self.assertEqual(result, expected) def test_route_with_version_number(self): # type: () -> None route1 = ApiRoute('alpha/get_metadata', 1, None) route1.set_attributes(None, None, Void(), Void(), Void(), {}) route2 = ApiRoute('alpha/get_metadata', 2, None) route2.set_attributes(None, None, Void(), Int32(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) result = self._evaluate_namespace(ns) expected = textwrap.dedent("""\ alpha_get_metadata = bb.Route( 'alpha/get_metadata', 1, False, bv.Void(), bv.Void(), bv.Void(), {}, ) alpha_get_metadata_v2 = bb.Route( 'alpha/get_metadata', 2, False, bv.Void(), bv.Int32(), bv.Void(), {}, ) ROUTES = { 'alpha/get_metadata': alpha_get_metadata, 'alpha/get_metadata:2': alpha_get_metadata_v2, } """) self.assertEqual(result, expected) def test_route_with_version_number_name_conflict(self): # type: () -> None route1 = ApiRoute('alpha/get_metadata', 2, None) route1.set_attributes(None, None, Void(), Int32(), Void(), {}) route2 = ApiRoute('alpha/get_metadata_v2', 1, None) route2.set_attributes(None, None, Void(), Void(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) with self.assertRaises(RuntimeError) as cm: self._evaluate_namespace(ns) self.assertEqual( 'There is a name conflict between {!r} and {!r}'.format(route1, route2), str(cm.exception)) def test_struct_with_custom_annotations(self): # type: () -> None ns = ApiNamespace('files') annotation_type = AnnotationType('MyAnnotationType', ns, None, [ AnnotationTypeParam('test_param', Int32(), None, False, None, None) ]) ns.add_annotation_type(annotation_type) annotation = CustomAnnotation('MyAnnotation', ns, None, 'MyAnnotationType', None, [], {'test_param': 42}) annotation.set_attributes(annotation_type) ns.add_annotation(annotation) struct = Struct('MyStruct', ns, None) struct.set_attributes(None, [ StructField('annotated_field', Int32(), None, None), StructField('unannotated_field', Int32(), None, None), ]) struct.fields[0].set_annotations([annotation]) struct.recursive_custom_annotations = set([annotation]) result = self._evaluate_struct(ns, struct) expected = textwrap.dedent('''\ class MyStruct(bb.Struct): __slots__ = [ '_annotated_field_value', '_unannotated_field_value', ] _has_required_fields = True def __init__(self, annotated_field=None, unannotated_field=None): self._annotated_field_value = bb.NOT_SET self._unannotated_field_value = bb.NOT_SET if annotated_field is not None: self.annotated_field = annotated_field if unannotated_field is not None: self.unannotated_field = unannotated_field # Instance attribute type: int (validator is set below) annotated_field = bb.Attribute("annotated_field") # Instance attribute type: int (validator is set below) unannotated_field = bb.Attribute("unannotated_field") def _process_custom_annotations(self, annotation_type, field_path, processor): super(MyStruct, self)._process_custom_annotations(annotation_type, field_path, processor) if annotation_type is MyAnnotationType: self.annotated_field = bb.partially_apply(processor, MyAnnotationType(test_param=42))('{}.annotated_field'.format(field_path), self.annotated_field) MyStruct_validator = bv.Struct(MyStruct) ''') # noqa self.maxDiff = None self.assertEqual(result, expected) def test_annotation_type_class(self): # type: () -> None ns = ApiNamespace('files') annotation_type = AnnotationType('MyAnnotationType', ns, "documented", [ AnnotationTypeParam( 'test_param', Int32(), "test parameter", False, None, None, ), AnnotationTypeParam( 'test_default_param', Int32(), None, True, 5, None, ), ]) result = self._evaluate_annotation_type(ns, annotation_type) expected = textwrap.dedent('''\ class MyAnnotationType(bb.AnnotationType): """ documented :ivar test_param: test parameter """ __slots__ = [ '_test_param', '_test_default_param', ] def __init__(self, test_param=None, test_default_param=5): self._test_param = test_param self._test_default_param = test_default_param @property def test_param(self): """ test parameter :rtype: int """ return self._test_param @property def test_default_param(self): """ :rtype: int """ return self._test_default_param ''') self.assertEqual(result, expected) # TODO: add more unit tests for client code generation stone-3.3.1/test/test_stone.py000077500000000000000000004607541417406541500164200ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals # pylint: disable=deprecated-method,useless-suppression import datetime import textwrap import unittest from stone.frontend.ast import ( AstNamespace, AstAlias, AstVoidField, AstTagRef, ) from stone.frontend.exception import InvalidSpec from stone.frontend.frontend import specs_to_ir from stone.frontend.parser import ParserFactory from stone.ir import ( Alias, is_boolean_type, is_integer_type, is_void_type, Nullable, RedactedBlot, RedactedHash, String, Map ) class TestStone(unittest.TestCase): """ Tests the Stone format. """ def setUp(self): self.parser_factory = ParserFactory(debug=False) def test_namespace_decl(self): text = textwrap.dedent("""\ namespace files """) out = self.parser_factory.get_parser().parse(text) self.assertIsInstance(out[0], AstNamespace) self.assertEqual(out[0].name, 'files') # test starting with newlines text = textwrap.dedent("""\ namespace files """) out = self.parser_factory.get_parser().parse(text) self.assertIsInstance(out[0], AstNamespace) self.assertEqual(out[0].name, 'files') def test_comments(self): text = textwrap.dedent("""\ # comment at top namespace files # another full line comment alias Rev = String # partial line comment struct S # comment before INDENT "Doc" # inner comment f1 UInt64 # partial line comment # f2 UInt64 # trailing comment struct S2 # struct def following comment # start with comment f1 String # end with partial-line comment # footer comment """) out = self.parser_factory.get_parser().parse(text) self.assertIsInstance(out[0], AstNamespace) self.assertIsInstance(out[1], AstAlias) self.assertEqual(out[2].name, 'S') self.assertEqual(out[3].name, 'S2') def test_line_continuations(self): line_continuation_err = 'Line continuation must increment indent by 1.' # Test continuation in various contexts text = textwrap.dedent("""\ namespace test alias U64 = UInt64( min_value=0, max_value=10) struct S val UInt64( min_value=0, max_value=10) val2 UInt64( # this # is # a min_value=0 # stress # test ) route r( S, S, S ) "Test route." """) specs_to_ir([('test.stone', text)]) # Try over indenting text = textwrap.dedent("""\ namespace test struct S val UInt64( # comment to throw it off min_value=0) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual(line_continuation_err, cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Try under indenting text = textwrap.dedent("""\ namespace test struct S val UInt64( # comment to throw it off # x2 min_value=0) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual(line_continuation_err, cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) def test_type_args(self): text = textwrap.dedent("""\ namespace test alias T = String(min_length=3) alias F = Float64(max_value=3.2e1) alias Numbers = List(UInt64) """) out = self.parser_factory.get_parser().parse(text) self.assertIsInstance(out[1], AstAlias) self.assertEqual(out[1].name, 'T') self.assertEqual(out[1].type_ref.name, 'String') self.assertEqual(out[1].type_ref.args[1]['min_length'], 3) self.assertIsInstance(out[2], AstAlias) self.assertEqual(out[2].name, 'F') self.assertEqual(out[2].type_ref.name, 'Float64') self.assertEqual(out[2].type_ref.args[1]['max_value'], 3.2e1) self.assertIsInstance(out[3], AstAlias) self.assertEqual(out[3].name, 'Numbers') self.assertEqual(out[3].type_ref.name, 'List') self.assertEqual(out[3].type_ref.args[0][0].name, 'UInt64') def test_struct_decl(self): # test struct decl with no docs text = textwrap.dedent("""\ namespace files struct QuotaInfo quota UInt64 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') # test struct with only a top-level doc text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." quota UInt64 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].doc, 'The space quota info for a user.') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') # test struct with field doc text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." quota UInt64 "The user's total quota allocation (bytes)." """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].doc, 'The space quota info for a user.') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') self.assertEqual(out[1].fields[0].doc, "The user's total quota allocation (bytes).") # test without newline after field doc text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." quota UInt64 "The user's total quota allocation (bytes)." """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].doc, 'The space quota info for a user.') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') self.assertEqual(out[1].fields[0].doc, "The user's total quota allocation (bytes).") # test with example text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." quota UInt64 "The user's total quota allocation (bytes)." example default quota=64000 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertIn('default', out[1].examples) # test with multiple examples text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." quota UInt64 "The user's total quota allocation (bytes)." example default quota=2000000000 example pro quota=100000000000 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertIn('default', out[1].examples) self.assertIn('pro', out[1].examples) # test with inheritance text = textwrap.dedent("""\ namespace test struct S1 f1 UInt64 struct S2 extends S1 f2 String """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'S1') self.assertEqual(out[2].name, 'S2') self.assertEqual(out[2].extends.name, 'S1') # test with defaults text = textwrap.dedent("""\ namespace ns struct S n1 Int32 = -5 n2 Int32 = 5 f1 Float64 = -1. f2 Float64 = -4.2 f3 Float64 = -5e-3 f4 Float64 = -5.1e-3 f5 Float64 = 1 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'S') self.assertEqual(out[1].fields[0].name, 'n1') self.assertTrue(out[1].fields[0].has_default) self.assertEqual(out[1].fields[0].default, -5) self.assertEqual(out[1].fields[1].default, 5) self.assertEqual(out[1].fields[2].default, -1) self.assertEqual(out[1].fields[3].default, -4.2) self.assertEqual(out[1].fields[4].default, -5e-3) self.assertEqual(out[1].fields[5].default, -5.1e-3) # float type should always have default value in float api = specs_to_ir([('test.stone', text)]) self.assertIsInstance(api.namespaces['ns'].data_type_by_name['S'].fields[6].default, float) # Try extending nullable type text = textwrap.dedent("""\ namespace test struct S f1 String struct S2 extends S? f2 String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Reference cannot be nullable.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) def test_struct_patch_decl(self): # test struct patch decl with no docs text = textwrap.dedent("""\ namespace files patch struct QuotaInfo quota UInt64 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') # test struct patch with a top-level doc text = textwrap.dedent("""\ namespace files patch struct QuotaInfo "The space quota info for a user." quota UInt64 """) out = self.parser_factory.get_parser().parse(text) msg, lineno, _ = self.parser_factory.get_parser().errors[0] # Can't parse patch with doc-string. self.assertEqual(msg, "Unexpected STRING with value 'The " + "space quota info for a user.'.") self.assertEqual(lineno, 3) # test struct patch with field doc text = textwrap.dedent("""\ namespace files patch struct QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertEqual(out[1].fields[0].name, 'quota') self.assertEqual(out[1].fields[0].type_ref.name, 'UInt64') self.assertEqual(out[1].fields[0].doc, "The user's total quota allocation (bytes).") # test with example text = textwrap.dedent("""\ namespace files patch struct QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." example default quota=64000 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertIn('default', out[1].examples) # test with multiple examples text = textwrap.dedent("""\ namespace files patch struct QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." example default quota=2000000000 example pro quota=100000000000 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'QuotaInfo') self.assertIn('default', out[1].examples) self.assertIn('pro', out[1].examples) # test with defaults text = textwrap.dedent("""\ namespace ns patch struct S n1 Int32 = -5 n2 Int32 = 5 f1 Float64 = -1. f2 Float64 = -4.2 f3 Float64 = -5e-3 f4 Float64 = -5.1e-3 """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'S') self.assertEqual(out[1].fields[0].name, 'n1') self.assertTrue(out[1].fields[0].has_default) self.assertEqual(out[1].fields[0].default, -5) self.assertEqual(out[1].fields[1].default, 5) self.assertEqual(out[1].fields[2].default, -1) self.assertEqual(out[1].fields[3].default, -4.2) self.assertEqual(out[1].fields[4].default, -5e-3) self.assertEqual(out[1].fields[5].default, -5.1e-3) # test no patching enumerated subtype text = textwrap.dedent("""\ namespace test struct Resource union file File folder Folder struct File extends Resource size UInt64 struct Folder extends Resource icon String struct Deleted extends Resource is_folder Boolean patch struct Resource union deleted Deleted """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Unexpected UNION with value 'union'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 18) def test_union_decl(self): # test union with only symbols text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." owner "Owner of a file." viewer "Read only permission." editor "Read and write permission." """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'Role') self.assertEqual(out[1].doc, 'The role a user may have in a shared folder.') self.assertIsInstance(out[1].fields[0], AstVoidField) self.assertEqual(out[1].fields[0].name, 'owner') self.assertIsInstance(out[1].fields[1], AstVoidField) self.assertEqual(out[1].fields[1].name, 'viewer') self.assertIsInstance(out[1].fields[2], AstVoidField) self.assertEqual(out[1].fields[2].name, 'editor') # TODO: Test a union that includes a struct. text = textwrap.dedent("""\ namespace files union Error A "Variant A" B "Variant B" """) self.parser_factory.get_parser().parse(text) # test with inheritance text = textwrap.dedent("""\ namespace test union U1 t1 UInt64 union U2 extends U1 t2 String """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'U1') self.assertEqual(out[2].name, 'U2') self.assertEqual(out[2].extends.name, 'U1') def test_union_patch_decl(self): # test union with only symbols text = textwrap.dedent("""\ namespace files patch union Role owner "Owner of a file." """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'Role') self.assertIsInstance(out[1].fields[0], AstVoidField) self.assertEqual(out[1].fields[0].name, 'owner') # test struct patch with a top-level doc text = textwrap.dedent("""\ namespace files patch union Role "The role a user may have in a shared folder." owner "Owner of a file." """) out = self.parser_factory.get_parser().parse(text) msg, lineno, _ = self.parser_factory.get_parser().errors[0] # Can't parse patch with doc-string. self.assertEqual(msg, "Unexpected STRING with value 'The " + "role a user may have in a shared folder.'.") self.assertEqual(lineno, 3) text = textwrap.dedent("""\ namespace files patch union Error A "Variant A" """) self.parser_factory.get_parser().parse(text) def test_composition(self): text = textwrap.dedent("""\ namespace files union UploadMode add overwrite struct Upload path String mode UploadMode = add """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[2].name, 'Upload') self.assertIsInstance(out[2].fields[1].default, AstTagRef) self.assertEqual(out[2].fields[1].default.tag, 'add') def test_route_decl(self): # Test route definition with no docstring text = textwrap.dedent("""\ namespace users route GetAccountInfo(Void, Void, Void) """) self.parser_factory.get_parser().parse(text) text = textwrap.dedent("""\ namespace users struct AccountInfo email String route GetAccountInfo(AccountInfo, Void, Void) "Gets the account info for a user" """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].name, 'AccountInfo') self.assertEqual(out[2].name, 'GetAccountInfo') self.assertEqual(out[2].arg_type_ref.name, 'AccountInfo') self.assertEqual(out[2].result_type_ref.name, 'Void') self.assertEqual(out[2].error_type_ref.name, 'Void') # Test raw documentation text = textwrap.dedent("""\ namespace users route GetAccountInfo(Void, Void, Void) "0 1 2 3 " """) out = self.parser_factory.get_parser().parse(text) self.assertEqual(out[1].doc, '0\n\n1\n\n2\n\n3\n') # Test deprecation text = textwrap.dedent("""\ namespace test route old_route (Void, Void, Void) deprecated """) api = specs_to_ir([('test.stone', text)]) r = api.namespaces['test'].route_by_name['old_route'] self.assertIsNotNone(r.deprecated) self.assertIsNone(r.deprecated.by) # Test deprecation with target route text = textwrap.dedent("""\ namespace test route old_route (Void, Void, Void) deprecated by new_route route new_route (Void, Void, Void) """) api = specs_to_ir([('test.stone', text)]) r_old = api.namespaces['test'].route_by_name['old_route'] r_new = api.namespaces['test'].route_by_name['new_route'] self.assertIsNotNone(r_old.deprecated) self.assertEqual(r_old.deprecated.by, r_new) # Test deprecation with target route (more complex route names) text = textwrap.dedent("""\ namespace test route test/old_route (Void, Void, Void) deprecated by test/new_route route test/new_route (Void, Void, Void) """) api = specs_to_ir([('test.stone', text)]) r_old = api.namespaces['test'].route_by_name['test/old_route'] r_new = api.namespaces['test'].route_by_name['test/new_route'] self.assertIsNotNone(r_old.deprecated) self.assertEqual(r_old.deprecated.by, r_new) # Try deprecation by undefined route text = textwrap.dedent("""\ namespace test route old_route (Void, Void, Void) deprecated by unk_route """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Undefined route 'unk_route' at version 1.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Try deprecation by struct text = textwrap.dedent("""\ namespace test route old_route (Void, Void, Void) deprecated by S struct S f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("'S' must be a route.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test route with version number and deprecation text = textwrap.dedent("""\ namespace test route get_metadata(Void, Void, Void) deprecated by get_metadata:2 route get_metadata:2(Void, Void, Void) """) api = specs_to_ir([('test.stone', text)]) routes = api.namespaces['test'].routes_by_name['get_metadata'] route_v1 = routes.at_version[1] route_v2 = routes.at_version[2] self.assertEqual(route_v1.name, 'get_metadata') self.assertEqual(route_v1.version, 1) self.assertEqual(route_v2.name, 'get_metadata') self.assertEqual(route_v2.version, 2) self.assertIsNotNone(route_v1.deprecated) self.assertEqual(route_v1.deprecated.by, route_v2) # Test using string as version number text = textwrap.dedent("""\ namespace test route get_metadata:beta(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Unexpected ID with value 'beta'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test using fraction as version number text = textwrap.dedent("""\ namespace test route get_metadata:1.2(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Unexpected FLOAT with value 1.2.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test using zero as version number text = textwrap.dedent("""\ namespace test route get_metadata:0(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Version number should be a positive integer.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test using negative integer as version number text = textwrap.dedent("""\ namespace test route get_metadata:-1(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Version number should be a positive integer.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test deprecating by a route at an undefined version text = textwrap.dedent("""\ namespace test route get_metadata(Void, Void, Void) deprecated by get_metadata:3 route get_metadata:2(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Undefined route 'get_metadata' at version 3.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test duplicate routes of same version text = textwrap.dedent("""\ namespace test route get_metadata:2(Void, Void, Void) route get_metadata:2(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Route 'get_metadata' at version 2 already defined (test.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 5) # Test user-friendly representation text = textwrap.dedent("""\ namespace test route alpha/get_metadata(Void, Void, Void) route alpha/get_metadata:2(Void, Void, Void) """) api = specs_to_ir([('test.stone', text)]) routes = api.namespaces['test'].routes_by_name['alpha/get_metadata'] route_v1 = routes.at_version[1] route_v2 = routes.at_version[2] self.assertEqual(route_v1.name_with_version(), 'alpha/get_metadata') self.assertEqual(route_v2.name_with_version(), 'alpha/get_metadata:2') def test_alphabetizing(self): text1 = textwrap.dedent("""\ namespace ns_b struct z f UInt64 union x a b struct y f UInt64 route b(Void, Void, Void) route a(Void, Void, Void) route c(Void, Void, Void) """) text2 = textwrap.dedent("""\ namespace ns_a route d (Void, Void, Void) """) api = specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) assert ['ns_a', 'ns_b'] == list(api.namespaces.keys()) ns_b = api.namespaces['ns_b'] assert [dt.name for dt in ns_b.data_types] == ['x', 'y', 'z'] assert [dt.name for dt in ns_b.routes] == ['a', 'b', 'c'] def test_lexing_errors(self): text = textwrap.dedent("""\ namespace users % # testing line numbers % struct AccountInfo email String """) parser = self.parser_factory.get_parser() out = parser.parse(text) msg, lineno = parser.lexer.errors[0] self.assertEqual(msg, "Illegal character '%'.") self.assertEqual(lineno, 4) msg, lineno = parser.lexer.errors[1] self.assertEqual(msg, "Illegal character '%'.") self.assertEqual(lineno, 8) # Check that despite lexing errors, parser marched on successfully. self.assertEqual(out[1].name, 'AccountInfo') text = textwrap.dedent("""\ namespace test struct S # Indent below is only 3 spaces f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Indent is not divisible by 4.", cm.exception.msg) def test_parsing_errors(self): text = textwrap.dedent("""\ namespace users strct AccountInfo email String """) parser = self.parser_factory.get_parser() parser.parse(text) msg, lineno, _ = parser.errors[0] self.assertEqual(msg, "Unexpected ID with value 'strct'.") self.assertEqual(lineno, 4) text = textwrap.dedent("""\ namespace users route test_route(Blah, Blah, Blah) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Symbol 'Blah' is undefined", cm.exception.msg) def test_name_clash(self): # namespace / type clash text = textwrap.dedent("""\ namespace test_namespace_test struct TestNamespaceTest str String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Name of user-defined type 'TestNamespaceTest' conflicts " "with name of namespace 'test_namespace_test'", cm.exception.msg) # namespace / route clash text = textwrap.dedent("""\ namespace test_namespace_test route test_namespace_test(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Name of route 'test_namespace_test' conflicts " "with name of namespace 'test_namespace_test'", cm.exception.msg) # namespace / alias clash text = textwrap.dedent("""\ namespace test_namespace_test alias TestNamespaceTest = String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Name of alias 'TestNamespaceTest' conflicts " "with name of namespace 'test_namespace_test'", cm.exception.msg) # route / type clash text = textwrap.dedent("""\ namespace test_namespace struct TestStructTest str String route test_struct_test(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Name of route 'test_struct_test' conflicts " "with name of user-defined type 'TestStructTest'", cm.exception.msg) # alias / route clash text = textwrap.dedent("""\ namespace test_namespace alias TestAliasTest = String route test_alias_test(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Name of route 'test_alias_test' conflicts " "with name of alias 'TestAliasTest'", cm.exception.msg) def test_docstrings(self): text = textwrap.dedent("""\ namespace test # No docstrings at all struct E f String struct S "Only type doc" f String struct T f String "Only field doc" union U "Only type doc" f String union V f String "Only field doc" # Check for inherited doc struct W extends T g String """) api = specs_to_ir([('test.stone', text)]) E_dt = api.namespaces['test'].data_type_by_name['E'] self.assertFalse(E_dt.has_documented_type_or_fields()) self.assertFalse(E_dt.has_documented_fields()) S_dt = api.namespaces['test'].data_type_by_name['S'] self.assertTrue(S_dt.has_documented_type_or_fields()) self.assertFalse(S_dt.has_documented_fields()) T_dt = api.namespaces['test'].data_type_by_name['T'] self.assertTrue(T_dt.has_documented_type_or_fields()) self.assertTrue(T_dt.has_documented_fields()) U_dt = api.namespaces['test'].data_type_by_name['U'] self.assertTrue(U_dt.has_documented_type_or_fields()) self.assertFalse(U_dt.has_documented_fields()) V_dt = api.namespaces['test'].data_type_by_name['V'] self.assertTrue(V_dt.has_documented_type_or_fields()) self.assertTrue(V_dt.has_documented_fields()) W_dt = api.namespaces['test'].data_type_by_name['W'] self.assertFalse(W_dt.has_documented_type_or_fields()) self.assertFalse(W_dt.has_documented_fields()) self.assertFalse(W_dt.has_documented_type_or_fields(), True) self.assertFalse(W_dt.has_documented_fields(), True) def test_alias(self): # Test aliasing to primitive text = textwrap.dedent("""\ namespace test alias R = String "This is a test of docstrings" """) api = specs_to_ir([('test.stone', text)]) test_ns = api.namespaces['test'] self.assertIsInstance(test_ns.aliases[0], Alias) self.assertEqual(test_ns.aliases[0].name, 'R') self.assertIsInstance(test_ns.aliases[0].data_type, String) self.assertEqual( test_ns.aliases[0].doc, 'This is a test of docstrings') # Test aliasing to primitive with additional attributes and nullable text = textwrap.dedent("""\ namespace test alias R = String(min_length=1)? """) api = specs_to_ir([('test.stone', text)]) test_ns = api.namespaces['test'] self.assertIsInstance(test_ns.aliases[0], Alias) self.assertEqual(test_ns.aliases[0].name, 'R') self.assertIsInstance(test_ns.aliases[0].data_type, Nullable) self.assertIsInstance(test_ns.aliases[0].data_type.data_type, String) # Test aliasing to alias text = textwrap.dedent("""\ namespace test alias T = String alias R = T """) api = specs_to_ir([('test.stone', text)]) test_ns = api.namespaces['test'] self.assertIsInstance(test_ns.alias_by_name['T'], Alias) self.assertIsInstance(test_ns.alias_by_name['R'], Alias) self.assertIsInstance(test_ns.alias_by_name['R'].data_type, Alias) self.assertEqual(test_ns.alias_by_name['R'].data_type.name, 'T') # Test order invariance text = textwrap.dedent("""\ namespace test alias R = T alias T = String """) api = specs_to_ir([('test.stone', text)]) # Try re-definition text = textwrap.dedent("""\ namespace test alias A = String alias A = UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Symbol 'A' already defined (test.stone:3).", cm.exception.msg) # Try cyclical reference text = textwrap.dedent("""\ namespace test alias A = B alias B = C alias C = A """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("Alias 'C' is part of a cycle.", cm.exception.msg) # Try aliasing to alias with attributes already set. text = textwrap.dedent("""\ namespace test alias T = String(min_length=1) alias R = T(min_length=1) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('Attributes cannot be specified for instantiated type', cm.exception.msg) # Test aliasing to composite and making it nullable text = textwrap.dedent("""\ namespace test struct S f String alias R = S? """) api = specs_to_ir([('test.stone', text)]) test_ns = api.namespaces['test'] S_dt = test_ns.data_type_by_name['S'] self.assertIsInstance(test_ns.alias_by_name['R'].data_type, Nullable) self.assertEqual(test_ns.alias_by_name['R'].data_type.data_type, S_dt) # Test aliasing to composite with attributes text = textwrap.dedent("""\ namespace test struct S f String alias R = S(min_length=1) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('Attributes cannot be specified for instantiated type', cm.exception.msg) # Test aliasing to route text = textwrap.dedent("""\ namespace test route test_route(Void, Void, Void) alias S = test_route """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("A route cannot be referenced here.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 5) self.assertEqual(cm.exception.path, 'test.stone') # Test aliasing from another namespace text1 = textwrap.dedent("""\ namespace test1 struct S f String """) text2 = textwrap.dedent("""\ namespace test2 import test1 alias S = test1.S """) api = specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) test1_ns = api.namespaces['test1'] S_dt = test1_ns.data_type_by_name['S'] test2_ns = api.namespaces['test2'] self.assertEqual(test2_ns.alias_by_name['S'].data_type, S_dt) # Try extending an alias-ed struct text1 = textwrap.dedent("""\ namespace test1 alias Z = S struct S f1 String struct T extends Z f2 String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) self.assertIn('A struct cannot extend an alias. Use the canonical name instead.', cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Try extending an alias-ed union text1 = textwrap.dedent("""\ namespace test1 alias Z = S union S f1 String union T extends Z f2 String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) self.assertIn( 'A union cannot extend an alias. Use the canonical name instead.', cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) def test_struct_semantics(self): # Test field with implicit void type text = textwrap.dedent("""\ namespace test struct S option_a """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Struct field 'option_a' cannot have a Void type.", cm.exception.msg) # Test duplicate fields text = textwrap.dedent("""\ namespace test struct A a UInt64 a String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('already defined', cm.exception.msg) # Test duplicate field name -- earlier being in a parent type text = textwrap.dedent("""\ namespace test struct A a UInt64 struct B extends A b String struct C extends B a String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('already defined in parent', cm.exception.msg) # Test extending from wrong type text = textwrap.dedent("""\ namespace test union A a struct B extends A b UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('struct can only extend another struct', cm.exception.msg) def test_union_semantics(self): # Test duplicate fields text = textwrap.dedent("""\ namespace test union_closed A a UInt64 a String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('already defined', cm.exception.msg) # Test duplicate field name -- earlier being in a parent type text = textwrap.dedent("""\ namespace test union_closed A a UInt64 union_closed B extends A b String union_closed C extends B a String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('already defined in parent', cm.exception.msg) # Test catch-all text = textwrap.dedent("""\ namespace test union A a b """) api = specs_to_ir([('test.stone', text)]) A_dt = api.namespaces['test'].data_type_by_name['A'] # Test both ways catch-all is exposed self.assertEqual(A_dt.catch_all_field, A_dt._fields_by_name['other']) self.assertTrue(A_dt._fields_by_name['other'].catch_all) # Try defining a child type as closed if its parent is open text = textwrap.dedent("""\ namespace test union A a union_closed B extends A b """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Union cannot be closed since parent type 'A' is open.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Try explicitly naming field "other" text = textwrap.dedent("""\ namespace test union A other """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Union cannot define an 'other' field because it is reserved as " "the catch-all field for open unions.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) # Test extending from wrong type text = textwrap.dedent("""\ namespace test struct A a UInt64 union B extends A b UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('union can only extend another union', cm.exception.msg) def test_map_semantics(self): text = textwrap.dedent("""\ namespace test alias M = Map(String, Int32) """) api = specs_to_ir([('test.stone', text)]) m_alias = api.namespaces['test'].alias_by_name['M'] self.assertIsInstance(m_alias, Alias) self.assertIsInstance(m_alias.data_type, Map) # maps of maps text = textwrap.dedent("""\ namespace test alias M = Map(String, Map(String, Int32)) """) api = specs_to_ir([('test.stone', text)]) m_alias = api.namespaces['test'].alias_by_name['M'] self.assertIsInstance(m_alias.data_type.value_data_type, Map) # Map type errors with 0 args text = textwrap.dedent("""\ namespace test alias M = Map() """) # map type errors with only 1 args with self.assertRaises(InvalidSpec): specs_to_ir([('test.stone', text)]) text = textwrap.dedent("""\ namespace test alias M = Map(String) """) # map type errors with more than two args with self.assertRaises(InvalidSpec): specs_to_ir([('test.stone', text)]) text = textwrap.dedent("""\ namespace test alias M = Map(String, String, String) """) with self.assertRaises(InvalidSpec): specs_to_ir([('test.stone', text)]) # map type errors when key data type is not a String text = textwrap.dedent("""\ namespace test alias M = Map(Int32, String) """) with self.assertRaises(InvalidSpec): specs_to_ir([('test.stone', text)]) def test_enumerated_subtypes(self): # Test correct definition text = textwrap.dedent("""\ namespace test struct Resource union file File folder Folder struct File extends Resource size UInt64 struct Folder extends Resource icon String """) specs_to_ir([('test.stone', text)]) # Test reference to non-struct text = textwrap.dedent("""\ namespace test struct Resource union file String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('must be a struct', cm.exception.msg) # Test reference to undefined type text = textwrap.dedent("""\ namespace test struct Resource union file File """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('Undefined', cm.exception.msg) # Test reference to non-subtype text = textwrap.dedent("""\ namespace test struct Resource union file File struct File size UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('not a subtype of', cm.exception.msg) # Test subtype listed more than once text = textwrap.dedent("""\ namespace test struct Resource union file File file2 File struct File extends Resource size UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn('only be specified once', cm.exception.msg) # Test missing subtype text = textwrap.dedent("""\ namespace test struct Resource union file File struct File extends Resource size UInt64 struct Folder extends Resource icon String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("missing 'Folder'", cm.exception.msg) # Test name conflict with field text = textwrap.dedent("""\ namespace test struct Resource union file File file String struct File extends Resource size UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("already defined on", cm.exception.msg) # Test if a leaf and its parent do not enumerate subtypes, but its # grandparent does. text = textwrap.dedent("""\ namespace test struct A union b B c String struct B extends A "No enumerated subtypes." struct C extends B "No enumerated subtypes." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("cannot be extended", cm.exception.msg) def unused_enumerated_subtypes_tests(self): # Currently, Stone does not allow for a struct that enumerates subtypes # to inherit from another struct that does. These tests only apply if # this restriction is removed. # Test name conflict with field in parent text = textwrap.dedent("""\ namespace test struct A union b B c String struct B extends A union c C struct C extends B d String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("already defined in parent", cm.exception.msg) # Test name conflict with union field in parent text = textwrap.dedent("""\ namespace test struct A union b B c String struct B extends A union b C struct C extends B d String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("already defined in parent", cm.exception.msg) # Test non-leaf with no enumerated subtypes text = textwrap.dedent("""\ namespace test struct A union b B c String struct B extends A "No enumerated subtypes." struct C extends B union d D struct D extends C e String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn("cannot enumerate subtypes if parent", cm.exception.msg) def test_nullable(self): # Test stacking nullable text = textwrap.dedent("""\ namespace test alias A = String? alias B = A? """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Cannot mark reference to nullable type as nullable.', cm.exception.msg) # Test stacking nullable text = textwrap.dedent("""\ namespace test alias A = String? struct S f A? """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Cannot mark reference to nullable type as nullable.', cm.exception.msg) # Test extending nullable text = textwrap.dedent("""\ namespace test struct S f String struct T extends S? g String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Reference cannot be nullable.', cm.exception.msg) def test_forward_reference(self): # Test route def before struct def text = textwrap.dedent("""\ namespace test route test_route(Void, S, Void) struct S f String """) specs_to_ir([('test.stone', text)]) # Test extending after... text = textwrap.dedent("""\ namespace test struct T extends S g String struct S f String """) specs_to_ir([('test.stone', text)]) # Test field ref to later-defined struct text = textwrap.dedent("""\ namespace test route test_route(Void, T, Void) struct T s S struct S f String """) specs_to_ir([('test.stone', text)]) # Test self-reference text = textwrap.dedent("""\ namespace test struct S s S? """) specs_to_ir([('test.stone', text)]) # Test forward union ref text = textwrap.dedent("""\ namespace test struct S s U = a union U a """) api = specs_to_ir([('test.stone', text)]) self.assertTrue(api.namespaces['test'].data_types[0].fields[0].has_default) self.assertEqual( api.namespaces['test'].data_types[0].fields[0].default.union_data_type, api.namespaces['test'].data_types[1]) self.assertEqual( api.namespaces['test'].data_types[0].fields[0].default.tag_name, 'a') def test_struct_patch_semantics(self): # Test patching normal struct text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch struct QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." example default quota=2000000000 example pro quota=100000000000 """) api = specs_to_ir([('test.stone', text)]) quota_info_dt = api.namespaces['files'].data_type_by_name['QuotaInfo'] self.assertEqual(quota_info_dt.fields[1].name, 'quota') self.assertTrue(is_integer_type(quota_info_dt.fields[1].data_type)) self.assertEqual(quota_info_dt.fields[1].doc, "The user's total quota allocation (bytes).") self.assertEqual(quota_info_dt.get_examples()['default'].value['quota'], 2000000000) self.assertEqual(quota_info_dt.get_examples()['pro'].value['quota'], 100000000000) # Test patching inherited struct text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" struct QuotaInfoPersonal extends QuotaInfo "The space quota info for a personal user." personal_quota UInt64 "The user's personal quota allocation (bytes)." patch struct QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." example default quota=2000000000 example pro quota=100000000000 """) api = specs_to_ir([('test.stone', text)]) quota_info_dt = api.namespaces['files'].data_type_by_name['QuotaInfoPersonal'] self.assertEqual(quota_info_dt.all_fields[1].name, 'quota') self.assertTrue(is_integer_type(quota_info_dt.all_fields[1].data_type)) self.assertEqual( quota_info_dt.all_fields[1].doc, "The user's total quota allocation (bytes).") # Testing patching the parent type of an enumerated subtype. text = textwrap.dedent("""\ namespace test struct Resource union file File folder Folder struct File extends Resource size UInt64 "The size of the file." example default size=5 struct Folder extends Resource icon String "The name of the icon." example default icon="My Icon" patch struct Resource is_public Boolean "Whether the resource is public." patch struct File example default is_public=true patch struct Folder example default is_public=false """) api = specs_to_ir([('test.stone', text)]) resource_dt = api.namespaces['test'].data_type_by_name['Resource'] self.assertEqual(resource_dt.all_fields[0].name, 'is_public') self.assertTrue(is_boolean_type(resource_dt.all_fields[0].data_type)) self.assertEqual(resource_dt.all_fields[0].doc, "Whether the resource is public.") file_dt = api.namespaces['test'].data_type_by_name['File'] self.assertEqual(file_dt.all_fields[0].name, 'is_public') self.assertTrue(is_boolean_type(file_dt.all_fields[0].data_type)) self.assertEqual(file_dt.all_fields[0].doc, "Whether the resource is public.") self.assertEqual(file_dt.get_examples()['default'].value['is_public'], True) folder_dt = api.namespaces['test'].data_type_by_name['Folder'] self.assertEqual(folder_dt.all_fields[0].name, 'is_public') self.assertTrue(is_boolean_type(file_dt.all_fields[0].data_type)) self.assertEqual(folder_dt.all_fields[0].doc, "Whether the resource is public.") self.assertEqual(folder_dt.get_examples()['default'].value['is_public'], False) # Try patching enumerated supertype struct with # nonnull field with missing example text = textwrap.dedent("""\ namespace test struct Resource union file File folder Folder struct File extends Resource size UInt64 "The size of the file." example default size=5 struct Folder extends Resource icon String "The name of the icon." example default icon="My Icon" patch struct Resource is_public Boolean "Whether the resource is public." patch struct File example default is_public=true """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Missing field 'is_public' in example.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 14) # Try patching type without pre-existing type text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch struct QuotaInfoDoesNotExist quota UInt64 "The user's total quota allocation (bytes)." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Patch 'QuotaInfoDoesNotExist' must correspond " + "to a pre-existing data_type.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Try patching type with pre-existing name of different type text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch union QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Type mismatch. Patch 'QuotaInfo' corresponds to " + "pre-existing data_type 'QuotaInfo' (test.stone:2) that has " + "type other than 'union'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Try patching union_closed type with pre-existing name of union type text = textwrap.dedent("""\ namespace files union_closed QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch union QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Type mismatch. Patch 'QuotaInfo' corresponds to " + "pre-existing data_type 'QuotaInfo' (test.stone:2) that has " + "type other than 'union_closed'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Try patching union type with pre-existing name of union_closed type text = textwrap.dedent("""\ namespace files union QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch union_closed QuotaInfo quota UInt64 "The user's total quota allocation (bytes)." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Type mismatch. Patch 'QuotaInfo' corresponds to " + "pre-existing data_type 'QuotaInfo' (test.stone:2) that has " + "type other than 'union'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Try patching field with pre-existing name text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch struct QuotaInfo user_id Int32 "The user associated with the quota." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Patched field 'user_id' overrides " + "pre-existing field in 'QuotaInfo' (test.stone:4).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 11) # Try patching field with example tag that doesn't exist text = textwrap.dedent("""\ namespace files struct QuotaInfo "The space quota info for a user." user_id String "The user associated with the quota." example default user_id="1234" example pro user_id="1234P" patch struct QuotaInfo quota UInt64 "The user associated with the quota." example doesNotExist user_id="1234" """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Example defined in patch 'QuotaInfo' must " + "correspond to a pre-existing example.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 13) def test_union_patch_semantics(self): # Test patching normal struct text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." viewer "Read only permission." editor "Read and write permission." patch union Role owner "Owner of a file." """) api = specs_to_ir([('test.stone', text)]) role_info_dt = api.namespaces['files'].data_type_by_name['Role'] self.assertEqual(role_info_dt.fields[2].name, 'owner') self.assertTrue(is_void_type(role_info_dt.fields[2].data_type)) self.assertEqual(role_info_dt.fields[2].doc, "Owner of a file.") # Test patching inherited union text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." viewer "Read only permission." editor "Read and write permission." union TeamRole extends Role "The team role a user may have in a shared folder." admin "Admin permission." patch union Role owner "Owner of a file." """) api = specs_to_ir([('test.stone', text)]) role_info_dt = api.namespaces['files'].data_type_by_name['TeamRole'] self.assertEqual(role_info_dt.all_fields[2].name, 'owner') self.assertTrue(is_void_type(role_info_dt.all_fields[2].data_type)) self.assertEqual(role_info_dt.all_fields[2].doc, "Owner of a file.") # Try patching type without pre-existing type text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." viewer "Read only permission." editor "Read and write permission." patch union RoleDoesNotExist owner "Owner of a file." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Patch 'RoleDoesNotExist' must correspond " + "to a pre-existing data_type.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Try patching type with pre-existing name of different type text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." viewer "Read only permission." editor "Read and write permission." patch struct Role owner String "Owner of a file." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Type mismatch. Patch 'Role' corresponds to " + "pre-existing data_type 'Role' (test.stone:2) that has " + "type other than 'struct'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Try patching field with pre-existing name text = textwrap.dedent("""\ namespace files union Role "The role a user may have in a shared folder." viewer "Read only permission." editor "Read and write permission." patch union Role viewer "Owner of a file." """) with self.assertRaises(InvalidSpec) as cm: api = specs_to_ir([('test.stone', text)]) self.assertEqual("Patched field 'viewer' overrides " + "pre-existing field in 'Role' (test.stone:4).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) def test_import(self): # Test field reference to another namespace ns1_text = textwrap.dedent("""\ namespace ns1 import ns2 struct S f ns2.S """) ns2_text = textwrap.dedent("""\ namespace ns2 struct S f String """) specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) # Test incorrectly importing the current namespace text = textwrap.dedent("""\ namespace test import test """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Cannot import current namespace.', cm.exception.msg) # Test importing a non-existent namespace text = textwrap.dedent("""\ namespace test import missingns """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Namespace 'missingns' is not defined in any spec.", cm.exception.msg) # Test extending struct from another namespace ns1_text = textwrap.dedent("""\ namespace ns1 import ns2 struct S extends ns2.T f String """) ns2_text = textwrap.dedent("""\ namespace ns2 struct T g String """) specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) # Test extending aliased struct from another namespace ns1_text = textwrap.dedent("""\ namespace ns1 import ns2 struct S extends ns2.X f String """) ns2_text = textwrap.dedent("""\ namespace ns2 alias X = T struct T g String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) self.assertEqual( 'A struct cannot extend an alias. Use the canonical name instead.', cm.exception.msg) # Test extending union from another namespace ns1_text = textwrap.dedent("""\ namespace ns1 import ns2 union V extends ns2.U b String """) ns2_text = textwrap.dedent("""\ namespace ns2 union U a """) specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) # Try circular import ns1_text = textwrap.dedent("""\ namespace ns1 import ns2 struct S t ns2.T """) ns2_text = textwrap.dedent("""\ namespace ns2 import ns1 struct T s ns1.S """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) self.assertIn( "Circular import of namespaces 'ns2' and 'ns1' detected.", cm.exception.msg) def test_doc_refs(self): # Test union doc referencing a field text = textwrap.dedent("""\ namespace test union U ":field:`a`" a b """) specs_to_ir([('test.stone', text)]) # Test union field doc referencing another field text = textwrap.dedent("""\ namespace test union U a ":field:`b`" b """) specs_to_ir([('test.stone', text)]) # Test union field doc referencing a field from an imported namespace text1 = textwrap.dedent("""\ namespace test1 union U a """) text2 = textwrap.dedent("""\ namespace test2 import test1 union U ":field:`test1.U.a`" b """) specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) # Test docs referencing a route text = textwrap.dedent("""\ namespace test route test_route(Void, Void, Void) struct T "type doc ref :route:`test_route`" f String "field doc ref :route:`test_route`" union U "type doc ref :route:`test_route`" f String "field doc ref :route:`test_route`" """) specs_to_ir([('test.stone', text)]) # Test docs referencing a route with version number text = textwrap.dedent("""\ namespace test route test_route:2(Void, Void, Void) struct T "type doc ref :route:`test_route:2`" f String "field doc ref :route:`test_route:2`" union U "type doc ref :route:`test_route:2`" f String "field doc ref :route:`test_route:2`" """) specs_to_ir([('test.stone', text)]) # Test referencing an undefined route text = textwrap.dedent("""\ namespace test struct T "type doc ref :route:`test_route`" f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Unknown doc reference to route 'test_route'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) self.assertEqual(cm.exception.path, 'test.stone') # Test referencing a field as a route text = textwrap.dedent("""\ namespace test union U a struct T "type doc ref :route:`U`" f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Doc reference to type 'U' is not a route.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) self.assertEqual(cm.exception.path, 'test.stone') # Test referencing a route at an undefined version text = textwrap.dedent("""\ namespace test route test_route:2(Void, Void, Void) struct T "type doc ref :route:`test_route:3`" f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Doc reference to route 'test_route' has undefined version 3.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) self.assertEqual(cm.exception.path, 'test.stone') # Test referencing a field of a route text = textwrap.dedent("""\ namespace test route test_route(Void, Void, Void) struct T "type doc ref :field:`test_route.g`" f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual("Bad doc reference to field 'g' of route 'test_route'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) self.assertEqual(cm.exception.path, 'test.stone') # Test referencing a field of alias text = textwrap.dedent("""\ namespace test struct T f String alias A = T struct B "ref to alias field :field:`A.f`." """) specs_to_ir([('test.stone', text)]) def test_namespace(self): # Test that namespace docstrings are combined ns1_text = textwrap.dedent("""\ namespace ns1 " This is a docstring for ns1. " struct S f String """) ns2_text = textwrap.dedent("""\ namespace ns1 " This is another docstring for ns1. " struct S2 f String """) api = specs_to_ir([('ns1.stone', ns1_text), ('ns2.stone', ns2_text)]) self.assertEqual( api.namespaces['ns1'].doc, 'This is a docstring for ns1.\nThis is another docstring for ns1.\n') def test_examples(self): # Test simple struct example text = textwrap.dedent("""\ namespace test struct S f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_types[0] self.assertTrue(s_dt.get_examples()['default'], {'f': 'A'}) # Test example with bad type text = textwrap.dedent("""\ namespace test struct S f String example default f = 5 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'f': integer is not a valid string", cm.exception.msg) # Test example with label "true". "false" and "null" are also # disallowed because they conflict with the identifiers for primitives. text = textwrap.dedent("""\ namespace test struct S f String example true f = "A" """) with self.assertRaises(InvalidSpec) as cm: # This raises an unexpected token error. specs_to_ir([('test.stone', text)]) # Test error case where two examples share the same label text = textwrap.dedent("""\ namespace test struct S f String example default f = "ZZZZZZ3" example default f = "ZZZZZZ4" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Example with label 'default' already defined on line 6.", cm.exception.msg) # Test error case where an example has the same field defined twice. text = textwrap.dedent("""\ namespace test struct S f String example default f = "ZZZZZZ3" f = "ZZZZZZ4" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Example with label 'default' defines field 'f' more than once.", cm.exception.msg) # Test empty examples text = textwrap.dedent("""\ namespace test struct S example default example other """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_types[0] self.assertIn('default', s_dt.get_examples()) self.assertIn('other', s_dt.get_examples()) self.assertNotIn('missing', s_dt.get_examples()) # Test missing field in example text = textwrap.dedent("""\ namespace test struct S f String example default """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Missing field 'f' in example.", cm.exception.msg) # Test missing default example text = textwrap.dedent("""\ namespace test struct S t T example default struct T f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Missing field 't' in example.", cm.exception.msg) # Test primitive field with default will use the default in the # example if it's missing. text = textwrap.dedent("""\ namespace test struct S f String = "S" example default """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_types[0] # Example should have no keys self.assertEqual(s_dt.get_examples()['default'].value['f'], 'S') # Test nullable primitive field missing from example text = textwrap.dedent("""\ namespace test struct S f String? example default """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_types[0] # Example should have no keys self.assertEqual(len(s_dt.get_examples()['default'].value), 0) # Test nullable primitive field explicitly set to null in example text = textwrap.dedent("""\ namespace test struct S f String? example default f = null """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_types[0] # Example should have no keys self.assertEqual(len(s_dt.get_examples()['default'].value), 0) # Test non-nullable primitive field explicitly set to null in example text = textwrap.dedent("""\ namespace test struct S f String example default f = null """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'f': null is not a valid string", cm.exception.msg) # Test example of composite type text = textwrap.dedent("""\ namespace test struct S t T example default t = default struct T f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'t': {'f': 'A'}}) # Test nullable composite missing from example text = textwrap.dedent("""\ namespace test struct S t T? example default t = default struct T f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'t': {'f': 'A'}}) # Test nullable composite explicitly set to null text = textwrap.dedent("""\ namespace test struct S t T? example default t = null struct T f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {}) # Test custom label text = textwrap.dedent("""\ namespace test struct S t T? example default t = special struct T f String r R example default f = "A" r = default example special f = "B" r = other struct R g String example default g = "D" example other g = "C" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'t': {'f': 'B', 'r': {'g': 'C'}}}) # Test missing label for composite example text = textwrap.dedent("""\ namespace test struct S t T? example default t = missing struct T f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Reference to example for 'T' with label 'missing' does not exist.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) # Test missing label for composite example text = textwrap.dedent("""\ namespace test struct S t T example default struct T f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Missing field 't' in example.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test bad label for composite example text = textwrap.dedent("""\ namespace test struct S t T? example default t = 34 struct T f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 't': example must reference label of 'T'", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) # Test solution for recursive struct # TODO: Omitting `s=null` will result in infinite recursion. text = textwrap.dedent("""\ namespace test struct S s S? f String example default f = "A" s = null """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'f': 'A'}) # Test examples with inheritance trees text = textwrap.dedent("""\ namespace test struct A a String struct B extends A b String struct C extends B c String example default a = "A" b = "B" c = "C" """) specs_to_ir([('test.stone', text)]) text = textwrap.dedent("""\ namespace test struct A a String struct B extends A b String struct C extends B c String example default b = "B" c = "C" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Missing field 'a' in example.", cm.exception.msg) def test_examples_union(self): # Test bad example with no fields specified text = textwrap.dedent("""\ namespace test union U a example default """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Example for union must specify exactly one tag.', cm.exception.msg) # Test bad example with more than one field specified text = textwrap.dedent("""\ namespace test union U a String b String example default a = "A" b = "B" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( 'Example for union must specify exactly one tag.', cm.exception.msg) # Test bad example with unknown tag text = textwrap.dedent("""\ namespace test union U a String example default z = "Z" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Unknown tag 'z' in example.", cm.exception.msg) # Test bad example with reference text = textwrap.dedent("""\ namespace test union U a String example default a = default """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'a': reference is not a valid string", cm.exception.msg) # Test bad example with null value for non-nullable text = textwrap.dedent("""\ namespace test union U a String example default a = null """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'a': null is not a valid string", cm.exception.msg) # Test example with null value for void type member text = textwrap.dedent("""\ namespace test union U a example default a = null """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 'a'}) self.assertEqual(u_dt.get_examples(compact=True)['default'].value, 'a') # Test simple union text = textwrap.dedent("""\ namespace test union U a b String c UInt64 example default b = "A" """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 'b', 'b': 'A'}) self.assertEqual(u_dt.get_examples()['a'].value, {'.tag': 'a'}) self.assertEqual(u_dt.get_examples(compact=True)['a'].value, 'a') self.assertNotIn('b', u_dt.get_examples()) # Test union with inheritance text = textwrap.dedent("""\ namespace test union U a String union V extends U b String example default a = "A" """) api = specs_to_ir([('test.stone', text)]) v_dt = api.namespaces['test'].data_type_by_name['V'] self.assertEqual(v_dt.get_examples()['default'].value, {'.tag': 'a', 'a': 'A'}) # Test union and struct text = textwrap.dedent("""\ namespace test union U a s S example default s = default example opt s = opt struct S f String example default f = "F" example opt f = "O" """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 's', 'f': 'F'}) self.assertEqual(u_dt.get_examples()['opt'].value, {'.tag': 's', 'f': 'O'}) self.assertEqual(list(u_dt.get_examples()['default'].value.keys())[0], '.tag') # Test union referencing non-existent struct example text = textwrap.dedent("""\ namespace test union U a s S example default s = missing struct S f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Reference to example for 'S' with label 'missing' does not exist.", cm.exception.msg) # Test fallback to union void member text = textwrap.dedent("""\ namespace test struct S u U example default u = a union U a b """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'u': {'.tag': 'a'}}) self.assertEqual(s_dt.get_examples(compact=True)['default'].value, {'u': 'a'}) # Test fallback to union member of composite type text = textwrap.dedent("""\ namespace test struct S u U example default u = default union U a b S2? example default b = default struct S2 f String example default f = "F" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'u': {'.tag': 'b', 'f': 'F'}}) # Test TagRef text = textwrap.dedent("""\ namespace test union U a b struct S u U = a example default """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'u': {'.tag': 'a'}}) self.assertEqual(s_dt.get_examples(compact=True)['default'].value, {'u': 'a'}) # Try TagRef to non-void option text = textwrap.dedent("""\ namespace test union U a UInt64 b struct S u U = a example default """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Field 'u' has an invalid default: invalid reference to non-void option 'a'", cm.exception.msg) # Try TagRef to non-existent option text = textwrap.dedent("""\ namespace test union U a UInt64 b struct S u U = c example default """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Field 'u' has an invalid default: invalid reference to unknown tag 'c'", cm.exception.msg) # Test bad void union member example value text = textwrap.dedent("""\ namespace test union U a example default a = false """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'a': example of void type must be null", cm.exception.msg) def test_examples_text(self): # Test multi-line example text (verify it gets unwrapp-ed) text = textwrap.dedent("""\ namespace test struct S a String example default "This is the text for the example. And I guess it's kind of long." a = "Hello, World." """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] example = s_dt.get_examples()['default'] self.assertEqual( example.text, "This is the text for the example. And I guess it's kind of long.") # Test union example text = textwrap.dedent("""\ namespace test union U a b String example default "This is the text for the example. And I guess it's kind of long." b = "Hi, World." """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] example = u_dt.get_examples()['default'] self.assertEqual( example.text, "This is the text for the example. And I guess it's kind of long.") def test_examples_enumerated_subtypes(self): # Test missing custom example text = textwrap.dedent("""\ namespace test struct S t T example other struct T f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Missing field 't' in example.", cm.exception.msg) # Test with two subtypes referenced text = textwrap.dedent("""\ namespace test struct R union s S t T a String example default s = default t = default struct S extends R b String struct T extends R c String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Example for struct with enumerated subtypes must only specify one subtype tag.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) # Test bad subtype reference text = textwrap.dedent("""\ namespace test struct R union s S t T a String example default s = 34 struct S extends R b String struct T extends R c String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Example of struct with enumerated subtypes must be a reference " "to a subtype's example.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Test unknown subtype text = textwrap.dedent("""\ namespace test struct R union s S t T a String example default z = default struct S extends R b String struct T extends R c String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Unknown subtype tag 'z' in example.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Test correct example of enumerated subtypes text = textwrap.dedent("""\ namespace test struct R union s S t T a String example default s = default struct S extends R b String example default a = "A" b = "B" struct T extends R c String """) api = specs_to_ir([('test.stone', text)]) r_dt = api.namespaces['test'].data_type_by_name['R'] self.assertEqual(r_dt.get_examples()['default'].value, {'.tag': 's', 'a': 'A', 'b': 'B'}) self.assertEqual(list(r_dt.get_examples()['default'].value.keys())[0], '.tag') # Test missing custom example text = textwrap.dedent("""\ namespace test struct R union s S t T a String example default s = default struct S extends R b String struct T extends R c String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Reference to example for 'S' with label 'default' does not exist.", cm.exception.msg) def test_examples_list(self): # Test field of list of primitives with bad example text = textwrap.dedent("""\ namespace test struct S l List(String) example default l = "a" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'l': string is not a valid list", cm.exception.msg) # Test field of list of primitives text = textwrap.dedent("""\ namespace test struct S l List(String) example default l = ["a", "b", "c"] """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': ['a', 'b', 'c']}) # Test nullable field of list of primitives text = textwrap.dedent("""\ namespace test struct S l List(String)? l2 List(String)? example default l = ["a", "b", "c"] l2 = null """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': ['a', 'b', 'c']}) # Test field of list of nullable primitives text = textwrap.dedent("""\ namespace test struct S l List(String?) example default l = ["a", null, "c"] """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': ['a', None, 'c']}) # Test example of list of composite types with bad example text = textwrap.dedent("""\ namespace test struct S l List(T) example default l = default struct T f String example default f = "A" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'l': reference is not a valid list", cm.exception.msg) # Test example of list of composite types text = textwrap.dedent("""\ namespace test struct S l List(T) l2 List(T)? l3 List(T)? example default l = [default, default] l2 = [default] l3 = null struct T f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': [{'f': 'A'}, {'f': 'A'}], 'l2': [{'f': 'A'}]}) # Test example of list of nullable composite types text = textwrap.dedent("""\ namespace test struct S l List(T?) example default l = [default, null] struct T f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': [{'f': 'A'}, None]}) # Test example of list of list of primitives text = textwrap.dedent("""\ namespace test struct S l List(List(String)) example default l = [["a", "b"], [], ["z"]] """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s_dt.get_examples()['default'].value, {'l': [['a', 'b'], [], ["z"]]}) # Test example of list of list of primitives with parameterization text = textwrap.dedent("""\ namespace test struct S l List(List(String, max_items=1)) example default l = [["a", "b"], [], ["z"]] """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'l': list has more than 1 item(s)", cm.exception.msg) # Test union with list (bad example) text = textwrap.dedent("""\ namespace test union U a List(String) example default a = "hi" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'a': string is not a valid list", cm.exception.msg) # Test union with list of primitives text = textwrap.dedent("""\ namespace test union U a List(String) example default a = ["hello", "world"] """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(s_dt.get_examples()['default'].value, {".tag": "a", 'a': ["hello", "world"]}) # Test union with list of composites text = textwrap.dedent("""\ namespace test union U a List(S) b List(S)? example default a = [default, default] example default_b b = [default] struct S f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) s_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(s_dt.get_examples()['default'].value, {'.tag': 'a', 'a': [{'f': 'A'}, {'f': 'A'}]}) self.assertEqual(s_dt.get_examples()['default_b'].value, {'.tag': 'b', 'b': [{'f': 'A'}]}) # Test union with list of lists of composites text = textwrap.dedent("""\ namespace test union U a List(List(S)) example default a = [[default]] struct S f String example default f = "A" """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 'a', 'a': [[{'f': 'A'}]]}) # Test union with list of list of primitives text = textwrap.dedent("""\ namespace test union U a List(List(String)) example default a = [["hello", "world"]] """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 'a', 'a': [['hello', 'world']]}) # Test union with list of primitives text = textwrap.dedent("""\ namespace test union U a List(List(String)) example default a = 42 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Bad example for field 'a': integer is not a valid list", cm.exception.msg) # Test union with list of list of structs text = textwrap.dedent("""\ namespace test union U a List(List(S)) example default a = [[default, special]] struct S a UInt64 example default a = 42 example special a = 100 """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual(u_dt.get_examples()['default'].value, {'.tag': 'a', 'a': [[{'a': 42}, {'a': 100}]]}) # Test union with list of list of unions text = textwrap.dedent("""\ namespace test union U a List(List(V)) example default a = [[default, special, x]] union V x y UInt64 example default x = null example special y = 100 """) api = specs_to_ir([('test.stone', text)]) u_dt = api.namespaces['test'].data_type_by_name['U'] self.assertEqual( u_dt.get_examples()['default'].value, {'.tag': 'a', 'a': [[{'.tag': 'x'}, {'.tag': 'y', 'y': 100}, {'.tag': 'x'}]]}) def test_examples_map(self): # valid simple example text = textwrap.dedent("""\ namespace test struct S m Map(String, Int32) example default m = {"one": 1, "two": 2} """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) # complex stone example text = textwrap.dedent("""\ namespace test alias m = Map(String, Int32) alias mm = Map(String, m) struct S arg mm "hash of hashes" example default arg = {"key": {"one": 1}, "another_key" : {"two" : 2, "three": 3}} """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) # map of structs text = textwrap.dedent("""\ namespace test struct Substruct m2 Map(String, Int32) example example_ref m2 = {"one": 1, "two": 2} struct S m Map(String, Substruct) example default m = {"key": example_ref, "another_key": example_ref} """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) # error when example doesn't match definition text = textwrap.dedent("""\ namespace test struct S m Map(String, String) example default m = {"one": 1} """) with self.assertRaises(InvalidSpec): specs_to_ir([('test.stone', text)]) # test multiline docstrings text = textwrap.dedent("""\ namespace test struct S m Map(String, Int32) example default m = { "one": 1, "two": 2 } """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) text = textwrap.dedent("""\ namespace test struct S m Map(String, Int32) example default m = { "one": 1, "two": 2 } """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) text = textwrap.dedent("""\ namespace test struct S m Map(String, Int32) example default m = { "one": 1, "two": 2 } """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) text = textwrap.dedent("""\ namespace test struct S m Map(String, Map(String, Int32)) example default m = { "one": { "one": 11, "two": 12 }, "two": { "one": 21, "two": 22 } } """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertIsInstance(s.get_examples()['default'].value, dict) def test_name_conflicts(self): # Test name conflict in same file text = textwrap.dedent("""\ namespace test struct S f String struct S g String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Symbol 'S' already defined (test.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test name conflict by route text = textwrap.dedent("""\ namespace test struct S f String route S (Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Symbol 'S' already defined (test.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test name conflict by union text = textwrap.dedent("""\ namespace test struct S f String union S g String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Symbol 'S' already defined (test.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test name conflict by alias text = textwrap.dedent("""\ namespace test struct S f String alias S = UInt64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Symbol 'S' already defined (test.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test name from two specs that are part of the same namespace text1 = textwrap.dedent("""\ namespace test struct S f String """) text2 = textwrap.dedent("""\ namespace test struct S f String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test1.stone', text1), ('test2.stone', text2)]) self.assertEqual( "Symbol 'S' already defined (test1.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) def test_imported_namespaces(self): text1 = textwrap.dedent("""\ namespace ns1 struct S1 f1 String struct S2 f2 String alias Iso8601 = Timestamp("%Y-%m-%dT%H:%M:%SZ") """) text2 = textwrap.dedent("""\ namespace ns2 import ns1 struct S3 f3 String f4 ns1.Iso8601? f5 ns1.S1? example default f3 = "hello" f4 = "2015-05-12T15:50:38Z" route r1(ns1.S1, ns1.S2, S3) """) api = specs_to_ir([('ns1.stone', text1), ('ns2.stone', text2)]) self.assertEqual(api.namespaces['ns2'].get_imported_namespaces(), [api.namespaces['ns1']]) xs = api.namespaces['ns2'].get_route_io_data_types() xs = sorted(xs, key=lambda x: x.name.lower()) self.assertEqual(len(xs), 3) ns1 = api.namespaces['ns1'] ns2 = api.namespaces['ns2'] self.assertEqual(xs[0].namespace, ns1) self.assertEqual(xs[1].namespace, ns1) s3_dt = ns2.data_type_by_name['S3'] self.assertEqual(s3_dt.fields[2].data_type.data_type.namespace, ns1) self.assertEqual(xs[2].name, 'S3') def test_namespace_obj(self): text = textwrap.dedent("""\ namespace ns1 struct S1 f1 String struct S2 f2 String s3 S3 struct S3 f3 String struct S4 f4 String alias A = S2 route r(S1, List(S4?)?, A) """) api = specs_to_ir([('ns1.stone', text)]) ns1 = api.namespaces['ns1'] # Check that all data types are defined self.assertIn('S1', ns1.data_type_by_name) self.assertIn('S2', ns1.data_type_by_name) self.assertIn('S3', ns1.data_type_by_name) self.assertIn('S4', ns1.data_type_by_name) self.assertEqual(len(ns1.data_types), 4) # Check that route is defined self.assertIn('r', ns1.route_by_name) self.assertEqual(len(ns1.routes), 1) s1 = ns1.data_type_by_name['S1'] a = ns1.alias_by_name['A'] s3 = ns1.data_type_by_name['S3'] s4 = ns1.data_type_by_name['S4'] route_data_types = ns1.get_route_io_data_types() self.assertIn(s1, route_data_types) # Test that aliased reference is included self.assertIn(a, route_data_types) # Test that field type is not present self.assertNotIn(s3, route_data_types) # Check that type that is wrapped by a list and/or nullable is present self.assertIn(s4, route_data_types) def test_whitespace(self): text = textwrap.dedent("""\ namespace test struct S f String ++++ g Int64 ++++ example default f = "hi" ++++++++ g = 3 route r(Void, S, Void) """).replace('+', ' ') specs_to_ir([('ns1.stone', text)]) text = textwrap.dedent("""\ namespace test struct S f String ++++ g Int64 ++++ example default f = "hi" ++++ ++++++ g = 3 route r(Void, S, Void) """).replace('+', ' ') specs_to_ir([('ns1.stone', text)]) text = textwrap.dedent("""\ namespace test # weirdly indented comment struct S # weirdly indented comment f String g Int64 example default f = "hi" # weirdly indented comment g = 3 route r(Void, S, Void) """) specs_to_ir([('ns1.stone', text)]) def test_route_attrs_schema(self): # Try to define route in stone_cfg stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String route r(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('stone_cfg.stone', stone_cfg_text)]) self.assertEqual( 'No routes can be defined in the stone_cfg namespace.', cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) self.assertEqual(cm.exception.path, 'stone_cfg.stone') # Try to set bad type for schema stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) attrs f1 = 3 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( 'integer is not a valid string', cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) self.assertEqual(cm.exception.path, 'test.stone') # Try missing attribute for route stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( "Route does not define attr key 'f1'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 2) self.assertEqual(cm.exception.path, 'test.stone') # Test missing attribute for route attribute with default stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String = "yay" """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) """) api = specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) ns1 = api.namespaces['test'] self.assertEqual(ns1.route_by_name['r1'].attrs['f1'], 'yay') # Test missing attribute for route attribute with optional stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String? """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) """) api = specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) test = api.namespaces['test'] self.assertEqual(test.route_by_name['r1'].attrs['f1'], None) # Test unknown route attributes stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 String? """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) attrs f2 = 3 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( "Route attribute 'f2' is not defined in 'stone_cfg.Route'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) self.assertEqual(cm.exception.path, 'test.stone') # Test no route attributes defined at all test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) attrs f1 = 3 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', test_text)]) self.assertEqual( "Route attribute 'f1' is not defined in 'stone_cfg.Route'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 4) self.assertEqual(cm.exception.path, 'test.stone') stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg struct Route f1 Boolean f2 Bytes f3 Float64 f4 Int64 f5 String f6 Timestamp("%Y-%m-%dT%H:%M:%SZ") f7 S f8 T f9 S? f10 T f11 S? alias S = String alias T = String? """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) attrs f1 = true f2 = "asdf" f3 = 3.2 f4 = 10 f5 = "Hello" f6 = "2015-05-12T15:50:38Z" f7 = "World" f8 = "World" f9 = "World" """) api = specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) test = api.namespaces['test'] attrs = test.route_by_name['r1'].attrs self.assertEqual(attrs['f1'], True) self.assertEqual(attrs['f2'], b'asdf') self.assertEqual(attrs['f3'], 3.2) self.assertEqual(attrs['f4'], 10) self.assertEqual(attrs['f5'], 'Hello') self.assertEqual( attrs['f6'], datetime.datetime(2015, 5, 12, 15, 50, 38)) self.assertEqual(attrs['f7'], 'World') self.assertEqual(attrs['f8'], 'World') self.assertEqual(attrs['f9'], 'World') self.assertEqual(attrs['f10'], None) self.assertEqual(attrs['f11'], None) # Try defining an attribute twice. stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg import test struct Route f1 String """) test_text = textwrap.dedent("""\ namespace test route r1(Void, Void, Void) attrs f1 = "1" f1 = "2" """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( "Attribute 'f1' defined more than once.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) self.assertEqual(cm.exception.path, 'test.stone') # Test union type stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg import test struct Route f1 test.U """) test_text = textwrap.dedent("""\ namespace test union U a b route r1(Void, Void, Void) attrs f1 = a """) specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) # Try union type with bad attribute stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg import test struct Route f1 test.U """) test_text = textwrap.dedent("""\ namespace test union U a b route r1(Void, Void, Void) attrs f1 = 3 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( "Expected union tag as value.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) self.assertEqual(cm.exception.path, 'test.stone') # Try union type attribute with non-void tag set stone_cfg_text = textwrap.dedent("""\ namespace stone_cfg import test struct Route f1 test.U """) test_text = textwrap.dedent("""\ namespace test union U a b String route r1(Void, Void, Void) attrs f1 = b """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([ ('stone_cfg.stone', stone_cfg_text), ('test.stone', test_text)]) self.assertEqual( "invalid reference to non-void option 'b'", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) self.assertEqual(cm.exception.path, 'test.stone') def test_inline_type_def(self): text = textwrap.dedent("""\ namespace test struct Photo dimensions Dimensions "Dimensions for a photo." struct height UInt64 "Height of the photo." width UInt64 "Width of the photo." example default height = 5 width = 10 location GpsCoordinates? struct latitude Float64 longitude Float64 example default latitude = 37.23 longitude = 122.2 time_taken Int64 "The timestamp when the photo was taken." example default "A typical photo" dimensions = default location = default time_taken = 100 union E e1 e2 E2 "Test E2." union a b route r(Void, Photo, E) """) specs_to_ir([('ns1.stone', text)]) text = textwrap.dedent("""\ namespace test struct T g Int64 struct S f T "Dimensions for a photo or video." struct a String b Int64 """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('ns1.stone', text)]) self.assertEqual( "Symbol 'T' already defined (ns1.stone:3).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) self.assertEqual(cm.exception.path, 'ns1.stone') def test_annotations(self): # Test non-existant annotation type text = textwrap.dedent("""\ namespace test annotation Broken = NonExistantType() """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Annotation type 'NonExistantType' does not exist", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test annotation that refers to something of the wrong type text = textwrap.dedent("""\ namespace test struct ItsAStruct f String annotation NonExistant = ItsAStruct() """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "'ItsAStruct' is not an annotation type", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test non-existant annotation text = textwrap.dedent("""\ namespace test struct S f String @NonExistant "Test field with a non-existant tag." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Annotation 'NonExistant' does not exist.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test omission tag text = textwrap.dedent("""\ namespace test annotation InternalOnly = Omitted("internal") struct S f String @InternalOnly "Test field with one omitted tag." """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s.all_fields[0].name, 'f') self.assertEqual(s.all_fields[0].omitted_caller, 'internal') # Test applying two omission tags to one field text = textwrap.dedent("""\ namespace test annotation InternalOnly = Omitted("internal") annotation AlphaOnly = Omitted("alpha_only") struct S f String @AlphaOnly @InternalOnly "Test field with two omitted tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Omitted caller already set as 'alpha_only'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test deprecated tag text = textwrap.dedent("""\ namespace test annotation Deprecated = Deprecated() struct S f String @Deprecated "Test field with one deprecated tag." """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s.all_fields[0].name, 'f') self.assertEqual(s.all_fields[0].deprecated, True) self.assertIn('Field is deprecated.', s.all_fields[0].doc) # Test applying two deprecated tags to one field text = textwrap.dedent("""\ namespace test annotation Deprecated = Deprecated() struct S f String @Deprecated @Deprecated "Test field with two deprecated tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Deprecated value already set as 'True'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) # Test preview tag text = textwrap.dedent("""\ namespace test annotation Preview = Preview() struct S f String @Preview "Test field with one preview tag." """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s.all_fields[0].name, 'f') self.assertEqual(s.all_fields[0].preview, True) self.assertIn('Field is in preview mode - do not rely on in production.', s.all_fields[0].doc) # Test applying two preview tags to one field text = textwrap.dedent("""\ namespace test annotation Preview = Preview() struct S f String @Preview @Preview "Test field with two preview tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Preview value already set as 'True'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) # Test applying both preview and deprecated (preview then deprecated) text = textwrap.dedent("""\ namespace test annotation Preview = Preview() annotation Deprecated = Deprecated() struct S f String @Preview @Deprecated "Test field with deprecated and preview tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn( "'Deprecated' and 'Preview' can't both be set.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test applying both preview and deprecated (deprecated then preview) text = textwrap.dedent("""\ namespace test annotation Preview = Preview() annotation Deprecated = Deprecated() struct S f String @Deprecated @Preview "Test field with deprecated and preview tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn( "'Deprecated' and 'Preview' can't both be set.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test redacted blot tag text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedBlot("test_regex") struct S f String @FieldRedactor "Test field with one redacted blot tag." """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s.all_fields[0].name, 'f') self.assertTrue(isinstance(s.all_fields[0].redactor, RedactedBlot)) self.assertEqual(s.all_fields[0].redactor.regex, "test_regex") # Test applying two redacted blot tags to one field text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedBlot("test_regex") annotation AnotherFieldRedactor = RedactedBlot("another_test_regex") struct S f String @FieldRedactor @AnotherFieldRedactor "Test field with two redacted blot tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn( "Redactor already set as \"RedactedBlot", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test redacted hash tag text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") struct S f UInt32 @FieldRedactor "Test field with one redacted hash tag." """) api = specs_to_ir([('test.stone', text)]) s = api.namespaces['test'].data_type_by_name['S'] self.assertEqual(s.all_fields[0].name, 'f') self.assertTrue(isinstance(s.all_fields[0].redactor, RedactedHash)) self.assertEqual(s.all_fields[0].redactor.regex, "test_regex") # Test applying two redacted blot tags to one field text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") annotation AnotherFieldRedactor = RedactedHash("another_test_regex") struct S f String @FieldRedactor @AnotherFieldRedactor "Test field with two redacted hash tags." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn( "Redactor already set as \"RedactedHash", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test redacted blot tag on alias text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedBlot("test_regex") alias TestAlias = UInt32 @FieldRedactor """) api = specs_to_ir([('test.stone', text)]) alias = api.namespaces['test'].alias_by_name['TestAlias'] self.assertTrue(isinstance(alias.redactor, RedactedBlot)) self.assertEqual(alias.redactor.regex, "test_regex") # Test redacted hash tag on alias text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") alias TestAlias = String @FieldRedactor """) api = specs_to_ir([('test.stone', text)]) alias = api.namespaces['test'].alias_by_name['TestAlias'] self.assertTrue(isinstance(alias.redactor, RedactedHash)) self.assertEqual(alias.redactor.regex, "test_regex") # Test deprecated tag (non-redact) tag on alias text = textwrap.dedent("""\ namespace test annotation Deprecated = Deprecated() alias TestAlias = String @Deprecated """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertIn( "Aliases only support 'Redacted' and custom annotations, not", cm.exception.msg) self.assertEqual(cm.exception.lineno, 5) # Test applying redactor tag to non-String/numeric field text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") struct T f String struct S f T? @FieldRedactor "Test field with non-String type." """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Redactors can't be applied to user-defined or void types.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 10) # Test applying redactor tag to non-String/numeric alias text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") struct T f String alias B = T alias A = B @FieldRedactor """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Redactors can't be applied to user-defined or void types.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) # Test applying redactor tag to alias to alias which already has redactor tag text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") alias B = String @FieldRedactor alias A = B @FieldRedactor """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "A redactor has already been defined for 'A' by 'B'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test applying redactor tag to alias field text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") alias A = String struct T f A @FieldRedactor """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Redactors can only be applied to alias definitions, not to alias references.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 8) # Test applying redactor tag list with user-defined object text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") struct S f String struct T f List(S) @FieldRedactor """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Redactors can't be applied to user-defined or void types.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) # Test applying redactor tag to map with user-defined object text = textwrap.dedent("""\ namespace test annotation FieldRedactor = RedactedHash("test_regex") struct S f String struct T f Map(String, S) @FieldRedactor """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Redactors can't be applied to user-defined or void types.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 9) def test_custom_annotations(self): # Test annotation type with non-primitive parameter text = textwrap.dedent("""\ namespace test struct S f String annotation_type AT s S """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Parameter 's' must have a primitive type (possibly nullable).", cm.exception.msg) self.assertEqual(cm.exception.lineno, 7) # Test annotation type with an annotation applied to a parameter text = textwrap.dedent("""\ namespace test annotation Deprecated = Deprecated() annotation_type AT s String @Deprecated """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Annotations cannot be applied to parameters of annotation types", cm.exception.msg) self.assertEqual(cm.exception.lineno, 6) # Test redefining built-in annotation types text = textwrap.dedent("""\ namespace test annotation_type Deprecated s String """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Cannot redefine built-in annotation type 'Deprecated'.", cm.exception.msg) self.assertEqual(cm.exception.lineno, 3) # Test not mixing keyword and positional arguments text = textwrap.dedent("""\ namespace test annotation_type Important owner String importance String = "very" annotation VeryImportant = Important("test-team", importance="very") """) with self.assertRaises(InvalidSpec) as cm: specs_to_ir([('test.stone', text)]) self.assertEqual( "Annotations accept either positional or keyword arguments, not both", cm.exception.msg, ) self.assertEqual(cm.exception.lineno, 6) # Test custom annotation type, instance, and application text = textwrap.dedent("""\ namespace test annotation_type Important owner String importance String = "very" annotation VeryImportant = Important("test-team") annotation SortaImportant = Important(owner="test-team", importance="sorta") alias TestAlias = String @VeryImportant struct TestStruct f String @SortaImportant g List(TestAlias) """) api = specs_to_ir([('test.stone', text)]) annotation_type = api.namespaces['test'].annotation_type_by_name['Important'] self.assertEqual(len(annotation_type.params), 2) self.assertEqual(annotation_type.params[0].name, 'owner') self.assertFalse(annotation_type.params[0].has_default) self.assertTrue(isinstance(annotation_type.params[0].data_type, String)) self.assertEqual(annotation_type.params[1].name, 'importance') self.assertTrue(annotation_type.params[1].has_default) self.assertEqual(annotation_type.params[1].default, 'very') self.assertTrue(isinstance(annotation_type.params[1].data_type, String)) # test both args and kwargs are set consistently in IR annotation = api.namespaces['test'].annotation_by_name['VeryImportant'] self.assertTrue(annotation.annotation_type is annotation_type) self.assertEqual(annotation.kwargs['owner'], 'test-team') self.assertEqual(annotation.kwargs['importance'], 'very') self.assertEqual(annotation.args[0], 'test-team') self.assertEqual(annotation.args[1], 'very') annotation = api.namespaces['test'].annotation_by_name['SortaImportant'] self.assertTrue(annotation.annotation_type is annotation_type) self.assertEqual(annotation.kwargs['owner'], 'test-team') self.assertEqual(annotation.kwargs['importance'], 'sorta') self.assertEqual(annotation.args[0], 'test-team') self.assertEqual(annotation.args[1], 'sorta') alias = api.namespaces['test'].alias_by_name['TestAlias'] self.assertTrue(alias.custom_annotations[0].annotation_type is annotation_type) struct = api.namespaces['test'].data_type_by_name['TestStruct'] self.assertEqual(struct.fields[0].custom_annotations[0], annotation) self.assertEqual(struct.recursive_custom_annotations, set([ (alias, api.namespaces['test'].annotation_by_name['VeryImportant']), (struct.fields[0], api.namespaces['test'].annotation_by_name['SortaImportant']), ])) # Test recursive references are captured ns2 = textwrap.dedent("""\ namespace testchain import test alias TestAliasChain = String @test.SortaImportant struct TestStructChain f test.TestStruct g List(TestAliasChain) """) ns3 = textwrap.dedent("""\ namespace teststruct import testchain struct TestStructToStruct f testchain.TestStructChain """) ns4 = textwrap.dedent("""\ namespace testalias import testchain struct TestStructToAlias f testchain.TestAliasChain """) api = specs_to_ir([('test.stone', text), ('testchain.stone', ns2), ('teststruct.stone', ns3), ('testalias.stone', ns4)]) struct_namespaces = [ns.name for ns in api.namespaces['teststruct'].get_imported_namespaces( consider_annotation_types=True)] self.assertTrue('test' in struct_namespaces) alias_namespaces = [ns.name for ns in api.namespaces['testalias'].get_imported_namespaces( consider_annotation_types=True)] self.assertTrue('test' in alias_namespaces) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_stone_internal.py000077500000000000000000000360411417406541500203000ustar00rootroot00000000000000#!/usr/bin/env python from __future__ import absolute_import, division, print_function, unicode_literals import unittest from stone.ir import ( ApiNamespace, Boolean, Float32, Float64, Int32, Int64, InvalidSpec, List, Map, ParameterError, String, Timestamp, UInt32, UInt64, Void, ) from stone.ir import ( Struct, StructField, Union, UnionField, ) from stone.frontend.ast import ( AstExample, AstExampleField, AstExampleRef, ) class TestStoneInternal(unittest.TestCase): """ Tests the internal representation of a Stone. """ def test_check_example(self): # # Test string # s = String(min_length=1, max_length=5) s.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value='hello', )) with self.assertRaises(InvalidSpec) as cm: s.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value='', )) self.assertIn("'' has fewer than 1 character(s)", cm.exception.msg) # # Test list # l1 = List(String(min_length=1), min_items=1, max_items=3) l1.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value=['asd'], )) with self.assertRaises(InvalidSpec) as cm: l1.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value=[], )) self.assertIn("has fewer than 1 item(s)", cm.exception.msg) # # Test list of lists # l1 = List(List(String(min_length=1), min_items=1)) l1.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value=[['asd']], )) with self.assertRaises(InvalidSpec) as cm: l1.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value=[[]], )) self.assertIn("has fewer than 1 item(s)", cm.exception.msg) # # Test Map type # m = Map(String(), String()) # valid example m.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value={"foo": "bar"} ) ) # does not conform to declared type with self.assertRaises(InvalidSpec): m.check_example( AstExampleField( path='test.stone', lineno=1, lexpos=0, name='v', value={1: "bar"} ) ) with self.assertRaises(ParameterError): # errors because only string types can be used as keys Map(Int32(), String()) s = Struct('S', None, None) s.set_attributes( "Docstring", [ StructField('a', UInt64(), 'a field', None), StructField('b', List(String()), 'a field', None), ], ) s._add_example( AstExample( 'test.stone', lineno=1, lexpos=0, label='default', text='Default example', fields={ 'a': AstExampleField( path='test.stone', lineno=2, lexpos=0, name='a', value=132, ), 'b': AstExampleField( path='test.stone', lineno=2, lexpos=0, name='b', value=['a'], ), } )) def test_string(self): s = String(min_length=1, max_length=3) # check correct str s.check('1') # check correct unicode s.check('\u2650') # check bad item with self.assertRaises(ValueError) as cm: s.check(99) self.assertIn('not a valid string', cm.exception.args[0]) # check too many characters with self.assertRaises(ValueError) as cm: s.check('12345') self.assertIn('more than 3 character(s)', cm.exception.args[0]) # check too few characters with self.assertRaises(ValueError) as cm: s.check('') self.assertIn('fewer than 1 character(s)', cm.exception.args[0]) def test_int(self): i = Int32() # check valid Int32 i.check(42) # check number that is too large with self.assertRaises(ValueError) as cm: i.check(2**31) self.assertIn('not within range', cm.exception.args[0]) # check number that is too small with self.assertRaises(ValueError) as cm: i.check(-2**31 - 1) self.assertIn('not within range', cm.exception.args[0]) i = UInt32() # check number that is too large with self.assertRaises(ValueError) as cm: i.check(2**32) self.assertIn('not within range', cm.exception.args[0]) # check number that is too small with self.assertRaises(ValueError) as cm: i.check(-1) self.assertIn('not within range', cm.exception.args[0]) i = Int64() # check number that is too large with self.assertRaises(ValueError) as cm: i.check(2**63) self.assertIn('not within range', cm.exception.args[0]) # check number that is too small with self.assertRaises(ValueError) as cm: i.check(-2**63 - 1) self.assertIn('not within range', cm.exception.args[0]) i = UInt64() # check number that is too large with self.assertRaises(ValueError) as cm: i.check(2**64) self.assertIn('not within range', cm.exception.args[0]) # check number that is too small with self.assertRaises(ValueError) as cm: i.check(-1) self.assertIn('not within range', cm.exception.args[0]) i = Int64(min_value=0, max_value=10) with self.assertRaises(ValueError) as cm: i.check(20) self.assertIn('20 is greater than 10', cm.exception.args[0]) with self.assertRaises(ValueError) as cm: i.check(-5) self.assertIn('-5 is less than 0', cm.exception.args[0]) # check that bad ranges are rejected self.assertRaises(ParameterError, lambda: Int64(min_value=0.1)) self.assertRaises(ParameterError, lambda: Int64(max_value='10')) def test_boolean(self): b = Boolean() # check valid bool b.check(True) # check non-bool with self.assertRaises(ValueError) as cm: b.check('true') self.assertIn('not a valid boolean', cm.exception.args[0]) def test_float(self): f = Float32() # check valid float f.check(3.14) # check non-float with self.assertRaises(ValueError) as cm: f.check('1.1') self.assertIn('not a valid real', cm.exception.args[0]) f = Float64(min_value=0, max_value=100) with self.assertRaises(ValueError) as cm: f.check(101) self.assertIn('is greater than', cm.exception.args[0]) with self.assertRaises(ParameterError) as cm: Float64(min_value=0, max_value=10**330) self.assertIn('too large for a float', cm.exception.args[0]) with self.assertRaises(ParameterError) as cm: Float32(min_value=0, max_value=10**50) self.assertIn('greater than the maximum value', cm.exception.args[0]) # check that bad ranges are rejected self.assertRaises(ParameterError, lambda: Float64(min_value=1j)) self.assertRaises(ParameterError, lambda: Float64(max_value='10')) def test_timestamp(self): t = Timestamp('%a, %d %b %Y %H:%M:%S') # check valid timestamp t.check('Sat, 21 Aug 2010 22:31:20') # check bad timestamp with self.assertRaises(ValueError) as cm: t.check('Sat, 21 Aug 2010') self.assertIn('does not match format', cm.exception.args[0]) def test_struct(self): ns = ApiNamespace('test') quota_info = Struct( 'QuotaInfo', None, ns, ) quota_info.set_attributes( "Information about a user's space quota.", [ StructField('quota', UInt64(), 'Total amount of space.', None), ], ) # add an example that doesn't fit the definition of a struct with self.assertRaises(InvalidSpec) as cm: quota_info._add_example( AstExample( path=None, lineno=None, lexpos=None, label='default', text=None, fields={ 'bad_field': AstExampleField( None, None, None, 'bad_field', 'xyz123')})) self.assertIn('has unknown field', cm.exception.msg) quota_info._add_example( AstExample( path=None, lineno=None, lexpos=None, label='default', text=None, fields={ 'quota': AstExampleField( None, None, None, 'quota', 64000)})) # set null for a required field with self.assertRaises(InvalidSpec) as cm: quota_info._add_example( AstExample( path=None, lineno=None, lexpos=None, label='null', text=None, fields={ 'quota': AstExampleField( None, None, None, 'quota', None)})) self.assertEqual( "Bad example for field 'quota': null is not a valid integer", cm.exception.msg) self.assertTrue(quota_info._has_example('default')) quota_info.nullable = True # test for structs within structs account_info = Struct( 'AccountInfo', None, ns, ) account_info.set_attributes( "Information about an account.", [ StructField('account_id', String(), 'Unique identifier for account.', None), StructField('quota_info', quota_info, 'Quota', None) ], ) account_info._add_example( AstExample( path=None, lineno=None, lexpos=None, label='default', text=None, fields={ 'account_id': AstExampleField( None, None, None, 'account_id', 'xyz123'), 'quota_info': AstExampleField( None, None, None, 'quota_info', AstExampleRef( None, None, None, 'default'))}) ) account_info._compute_examples() # ensure that an example for quota_info is propagated up self.assertIn('quota_info', account_info.get_examples()['default'].value) def test_union(self): ns = ApiNamespace('files') update_parent_rev = Struct( 'UpdateParentRev', None, ns, ) update_parent_rev.set_attributes( "Overwrite existing file if the parent rev matches.", [ StructField('parent_rev', String(), 'The revision to be updated.', None) ], ) update_parent_rev._add_example( AstExample( path=None, lineno=None, lexpos=None, label='default', text=None, fields={ 'parent_rev': AstExampleField( None, None, None, 'parent_rev', 'xyz123')})) # test variants with only tags, as well as those with structs. conflict = Union( 'WriteConflictPolicy', None, ns, True, ) conflict.set_attributes( 'Policy for managing write conflicts.', [ UnionField( 'reject', Void(), 'On a write conflict, reject the new file.', None), UnionField( 'overwrite', Void(), 'On a write conflict, overwrite the existing file.', None), UnionField( 'update_if_matching_parent_rev', update_parent_rev, 'On a write conflict, overwrite the existing file.', None), ], ) conflict._add_example( AstExample( path=None, lineno=None, lexpos=None, label='default', text=None, fields={ 'update_if_matching_parent_rev': AstExampleField( None, None, None, 'update_if_matching_parent_rev', AstExampleRef(None, None, None, 'default'))})) conflict._compute_examples() # test that only null value is returned for an example of a Void type self.assertEqual(conflict.get_examples()['reject'].value, {'.tag': 'reject'}) # test that dict is returned for a tagged struct variant self.assertEqual(conflict.get_examples()['default'].value, {'.tag': 'update_if_matching_parent_rev', 'parent_rev': 'xyz123'}) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_stone_route_whitelist.py000066400000000000000000000325611417406541500217160ustar00rootroot00000000000000#!/usr/bin/env python import textwrap import unittest from stone.frontend.frontend import specs_to_ir class TestStone(unittest.TestCase): def _compare_namespace_names(self, api, expected_names): expected_names = sorted(expected_names) actual_names = sorted(list(api.namespaces.keys())) self.assertEqual(expected_names, actual_names) def _compare_datatype_names(self, namespace, expected_names): expected_names = sorted(expected_names) actual_names = sorted(list(namespace.data_type_by_name.keys())) self.assertEqual(expected_names, actual_names) actual_names_on_datatypes = sorted([d.name for d in namespace.data_type_by_name.values()]) self.assertEqual(actual_names, actual_names_on_datatypes) def test_simple(self): """ Tests that route whitelisting can generate the right datatypes for a namespace. """ text = textwrap.dedent("""\ namespace test struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" route TestRoute (TestArg, TestResult, Void) "test doc" struct TestArg2 f String "test doc" example default f = "asdf" struct TestResult2 f String "test doc" example default f = "asdf" route TestRoute:2 (TestArg2, TestResult2, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["TestRoute:2"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg2', 'TestResult2']) def test_stable_ordering(self): """ Tests that route generation returns the same route order on generation. """ text = textwrap.dedent("""\ namespace test struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" route TestRoute (TestArg, TestResult, Void) "test doc" route OtherRoute (Void, Void, Void) "Additional Route" """) route_whitelist = ["OtherRoute", "TestRoute"] for x in range(2, 100): text += textwrap.dedent("""\ route TestRoute:{} (TestArg, TestResult, Void) "test doc" """.format(x)) route_whitelist.append("TestRoute:{}".format(x)) route_whitelist_filter = { "route_whitelist": {"test": route_whitelist}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) routes = api.namespaces['test'].routes self.assertEqual(len(routes), 100) self.assertEqual(routes[0].name, "OtherRoute") for x in range(1, 100): self.assertEqual(routes[x].name, "TestRoute") self.assertEqual(routes[x].version, x) def test_star(self): """ Tests that inputs with "*" work as expected. """ text = textwrap.dedent("""\ namespace test struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" route TestRoute (TestArg, TestResult, Void) "test doc" struct TestArg2 f String "test doc" example default f = "asdf" struct TestResult2 f String "test doc" example default f = "asdf" route TestRoute:2 (TestArg2, TestResult2, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["*"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestArg2', 'TestResult', 'TestResult2']) def test_alias(self): """ Tests that aliased datatypes are correctly generated. """ text = textwrap.dedent("""\ namespace test struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" alias TestAlias = TestArg route TestRoute (TestAlias, TestResult, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestResult']) def test_imports(self): """ Tests that datatypes imported from another namespace are correctly included. """ text = textwrap.dedent("""\ namespace test struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" """) text2 = textwrap.dedent("""\ namespace test2 import test route TestRoute (test.TestArg, test.TestResult, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test2": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text), ('test2.stone', text2)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test', 'test2']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestResult']) self._compare_datatype_names(api.namespaces['test2'], []) def test_builtin_types(self): """ Tests that builtin datatypes, like lists, maps, and unions, are correctly evaluated. """ text = textwrap.dedent("""\ namespace test union Foo a "test doc" example default a = null union Bar a "test doc" example default a = null struct TestArg f String "test doc" example default f = "asdf" struct TestResult f List(Foo) "test doc" b Map(String, Bar) "test doc" example default f = [default] b = {"test": default} route TestRoute (TestArg, TestResult, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestResult', 'Foo', 'Bar']) def test_subtype(self): """ Tests that datatypes that inherit from others are correctly generated. """ text = textwrap.dedent("""\ namespace test union Foo a "test doc" example default a = null union Bar extends Foo b "test doc" example default a = null struct TestArg f String "test doc" example default f = "asdf" struct TestResult f Bar "test doc" example default f = default route TestRoute (TestArg, TestResult, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestResult', 'Foo', 'Bar']) # Test enumerated subtypes as well text = textwrap.dedent("""\ namespace test struct Foo union file File folder Folder path String "test doc" example default file = default example folder_default folder = default struct File extends Foo a String "test doc" example default a = "a" path = "a" struct Folder extends Foo b String "test doc" example default b = "b" path = "a" struct TestArg f Foo "test doc" example default f = default struct TestResult f String "test doc" example default f = "asdf" route TestRoute (TestArg, TestResult, Void) "test doc" """) route_whitelist_filter = { "route_whitelist": {"test": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test']) self._compare_datatype_names(api.namespaces['test'], ['TestArg', 'TestResult', 'Foo', 'File', 'Folder']) def test_doc_refs(self): """ Tests that datatypes referenced in documentation get generated. """ text = textwrap.dedent("""\ namespace test union Foo a "test doc" example default a = null union Bar a "test doc" example default a = null """) text2 = textwrap.dedent("""\ namespace test2 ":type:`test.Foo`" alias TestAlias = String ":field:`Baz.a`" struct TestStruct a TestAlias "test doc" example default a = "asdf" struct Baz a String "test doc :field:`TestStruct.a`" example default a = "asdf" struct TestArg f String "test doc" example default f = "asdf" struct TestResult f String "test doc" example default f = "asdf" route TestRoute (TestArg, TestResult, Void) ":type:`Baz` test doc" """) route_whitelist_filter = { "route_whitelist": {"test2": ["TestRoute"]}, "datatype_whitelist": {} } api = specs_to_ir([('test.stone', text), ('test2.stone', text2)], route_whitelist_filter=route_whitelist_filter) self._compare_namespace_names(api, ['test', 'test2']) self._compare_datatype_names(api.namespaces['test'], ['Foo']) self._compare_datatype_names(api.namespaces['test2'], ['TestArg', 'TestResult', 'TestStruct', 'Baz']) if __name__ == '__main__': unittest.main() stone-3.3.1/test/test_tsd_client.py000066400000000000000000000103321417406541500173740ustar00rootroot00000000000000import textwrap import unittest from stone.backends.tsd_client import TSDClientBackend from stone.ir import Api, ApiNamespace, ApiRoute, Void, Int32 from stone.ir.data_types import Struct MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression class TestGeneratedTSDClient(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestGeneratedTSDClient, self).__init__(*args, **kwargs) def _get_api(self): # type () -> Api api = Api(version='0.1b1') api.route_schema = Struct('Route', 'stone_cfg', None) route1 = ApiRoute('get_metadata', 1, None) route1.set_attributes(None, ':route:`get_metadata`', Void(), Void(), Void(), {}) route2 = ApiRoute('get_metadata', 2, None) route2.set_attributes(None, ':route:`get_metadata:2`', Void(), Int32(), Void(), {}) route3 = ApiRoute('get_metadata', 3, None) route3.set_attributes(None, ':route:`get_metadata:3`', Int32(), Int32(), Void(), {}) ns = ApiNamespace('files') ns.add_route(route1) ns.add_route(route2) ns.add_route(route3) api.namespaces[ns.name] = ns return api, ns def test__generate_types_single_ns(self): # type: () -> None api, _ = self._get_api() backend = TSDClientBackend( target_folder_path="output", args=['files', 'files'] ) backend._generate_routes(api, 0, 0) result = backend.output_buffer_to_string() expected = textwrap.dedent( '''\ /** * getMetadata() * * When an error occurs, the route rejects the promise with type Error. */ public filesGetMetadata(): Promise; /** * getMetadataV2() * * When an error occurs, the route rejects the promise with type Error. */ public filesGetMetadataV2(): Promise; /** * getMetadataV3() * * When an error occurs, the route rejects the promise with type Error. * @param arg The request parameters. */ public filesGetMetadataV3(arg: number): Promise; ''') self.assertEqual(result, expected) def test__generate_types_with_wrap_response_flag(self): # type: () -> None api, _ = self._get_api() backend = TSDClientBackend( target_folder_path="output", args=['files', 'files', '--wrap-response-in', 'DropboxResponse'] ) backend._generate_routes(api, 0, 0) result = backend.output_buffer_to_string() expected = textwrap.dedent( '''\ /** * getMetadata() * * When an error occurs, the route rejects the promise with type Error. */ public filesGetMetadata(): Promise>; /** * getMetadataV2() * * When an error occurs, the route rejects the promise with type Error. */ public filesGetMetadataV2(): Promise>; /** * getMetadataV3() * * When an error occurs, the route rejects the promise with type Error. * @param arg The request parameters. */ public filesGetMetadataV3(arg: number): Promise>; ''') self.assertEqual(result, expected) def test_route_with_version_number_conflict(self): # type: () -> None api, ns = self._get_api() # Add a conflicting route route3 = ApiRoute('get_metadata_v2', 1, None) route3.set_attributes(None, None, Void(), Int32(), Void(), {}) ns.add_route(route3) backend = TSDClientBackend( target_folder_path="output", args=['files', 'files'] ) with self.assertRaises(RuntimeError) as cm: backend._generate_routes(api, 0, 0) self.assertTrue(str(cm.exception).startswith( 'There is a name conflict between')) stone-3.3.1/test/test_tsd_types.py000066400000000000000000000352641417406541500172750ustar00rootroot00000000000000from __future__ import absolute_import, division, print_function, unicode_literals import textwrap MYPY = False if MYPY: import typing # noqa: F401 # pylint: disable=import-error,unused-import,useless-suppression import os import unittest import subprocess import sys import shutil try: # Works for Py 3.3+ from unittest.mock import Mock except ImportError: # See https://github.com/python/mypy/issues/1153#issuecomment-253842414 from mock import Mock # type: ignore from stone.ir import ( ApiNamespace, Boolean, Struct, StructField) from stone.backends.tsd_types import TSDTypesBackend from test.backend_test_util import _mock_output def _make_backend(target_folder_path, template_path, custom_args=None): # type: (typing.Text, typing.Text, typing.List) -> TSDTypesBackend args = Mock() arg_values = [template_path, "-i=0"] if custom_args: arg_values = arg_values + custom_args args.__iter__ = Mock(return_value=iter(arg_values)) return TSDTypesBackend( target_folder_path=str(target_folder_path), args=args ) def _make_namespace(ns_name="accounts"): # type: (typing.Text) -> ApiNamespace ns = ApiNamespace(ns_name) struct = _make_struct('User', 'exists', ns) ns.add_data_type(struct) return ns def _make_struct(struct_name, struct_field_name, namespace): # type: (typing.Text, typing.Text, ApiNamespace) -> Struct struct = Struct(name=struct_name, namespace=namespace, ast_node=None) struct.set_attributes(None, [StructField(struct_field_name, Boolean(), None, None)]) return struct def _evaluate_namespace(backend, namespace_list): # type: (TSDTypesBackend, typing.List[ApiNamespace]) -> typing.Text get_result = _mock_output(backend) filename = "types.d.ts" backend.split_by_namespace = False backend._generate_base_namespace_module(namespace_list=namespace_list, filename=filename, extra_args={}, template="""/*TYPES*/""", exclude_error_types=True) return get_result() class TestTSDTypes(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestTSDTypes, self).__init__(*args, **kwargs) self.maxDiff = None # Increase text diff size def test__generate_types_single_ns(self): # type: () -> None backend = _make_backend(target_folder_path="output", template_path="") ns = _make_namespace() result = _evaluate_namespace(backend, [ns]) expected = textwrap.dedent(""" type Timestamp = string; namespace accounts { export interface User { exists: boolean; } } """) self.assertEqual(result, expected) def test__generate_types_empty_ns(self): # type: () -> None backend = _make_backend(target_folder_path="output", template_path="") empty_ns = ApiNamespace("empty_namespace") result = _evaluate_namespace(backend, [empty_ns]) expected = textwrap.dedent("") self.assertEqual(result, expected) def test__generate_types_with_empty_ns(self): # type: () -> None backend = _make_backend(target_folder_path="output", template_path="") ns = _make_namespace() empty_ns = ApiNamespace("empty_namespace") result = _evaluate_namespace(backend, [ns, empty_ns]) expected = textwrap.dedent(""" type Timestamp = string; namespace accounts { export interface User { exists: boolean; } } """) self.assertEqual(result, expected) def test__generate_types_multiple_ns(self): # type: () -> None backend = _make_backend(target_folder_path="output", template_path="") ns1 = _make_namespace("accounts") ns2 = _make_namespace("files") result = _evaluate_namespace(backend, [ns1, ns2]) expected = textwrap.dedent(""" type Timestamp = string; namespace accounts { export interface User { exists: boolean; } } namespace files { export interface User { exists: boolean; } } """) self.assertEqual(result, expected) def test__generate_types_multiple_ns_with_export(self): # type: () -> None backend = _make_backend(target_folder_path="output", template_path="", custom_args=["--export-namespaces"]) ns1 = _make_namespace("accounts") ns2 = _make_namespace("files") result = _evaluate_namespace(backend, [ns1, ns2]) expected = textwrap.dedent(""" type Timestamp = string; export namespace accounts { export interface User { exists: boolean; } } export namespace files { export interface User { exists: boolean; } } """) self.assertEqual(result, expected) class SpecHelper: """ A helper class which exposes two namespace definitions and its corresponding type definitions for testing. The types are available as either a declaration or a namespace. """ def __init__(self): pass _error_types = """ /** * An Error object returned from a route. */ interface Error { // Text summary of the error. error_summary: string; // The error object. error: T; // User-friendly error message. user_message: UserMessage; } /** * User-friendly error message. */ interface UserMessage { // The message. text: string; // The locale of the message. locale: string; } """ _ns_spec = """\ namespace ns import ns2 struct A "Sample struct doc." a String "Sample field doc." b Int64 struct B extends ns2.BaseS c Bytes """ _ns_spec_types = """{ /** * Sample struct doc. */ export interface A { /** * Sample field doc. */ a: string; b: number; } export interface B extends ns2.BaseS { c: string; } %s } """ _ns2_spec = """\ namespace ns2 struct BaseS "This is a test." z Int64 maptype Map(String, Int64) union_closed BaseU z x String alias AliasedBaseU = BaseU """ _ns2_spec_types = """{ /** * This is a test. */ export interface BaseS { z: number; maptype: {[key: string]: number}; } export interface BaseUZ { '.tag': 'z'; } export interface BaseUX { '.tag': 'x'; x: string; } export type BaseU = BaseUZ | BaseUX; export type AliasedBaseU = BaseU; %s } """ _ns3_union_spec = """\ namespace ns3 struct A union a1 A1 a2 A2 a String struct A1 extends A b Boolean struct A2 extends A c Boolean mapfield Map(String, A) union M e Boolean f String union B w Boolean x A y M z A2 """ _ns3_union_spec_types = """{ export interface A { a: string; } /** * Reference to the A polymorphic type. Contains a .tag property to let you * discriminate between possible subtypes. */ export interface AReference extends A { /** * Tag identifying the subtype variant. */ '.tag': "a1"|"a2"; } export interface A1 extends A { b: boolean; } /** * Reference to the A1 type, identified by the value of the .tag property. */ export interface A1Reference extends A1 { /** * Tag identifying this subtype variant. This field is only present when * needed to discriminate between multiple possible subtypes. */ '.tag': 'a1'; } export interface A2 extends A { c: boolean; mapfield: {[key: string]: A}; } /** * Reference to the A2 type, identified by the value of the .tag property. */ export interface A2Reference extends A2 { /** * Tag identifying this subtype variant. This field is only present when * needed to discriminate between multiple possible subtypes. */ '.tag': 'a2'; } export interface BW { '.tag': 'w'; w: boolean; } export interface BX { '.tag': 'x'; x: A1Reference|A2Reference|AReference; } export interface BY { '.tag': 'y'; y: M; } export interface BZ extends A2 { '.tag': 'z'; } export interface BOther { '.tag': 'other'; } export type B = BW | BX | BY | BZ | BOther; export interface ME { '.tag': 'e'; e: boolean; } export interface MF { '.tag': 'f'; f: string; } export interface MOther { '.tag': 'other'; } export type M = ME | MF | MOther; %s } """ _timestamp_mapping = 'type Timestamp = string' _timestamp_def_formatted = "\n" + " " + _timestamp_mapping + ";" @classmethod def get_ns_spec(cls): """Returns a test namespace which imports another namespace (`ns2`).""" return cls._ns_spec @classmethod def get_ns_types_as_declaration(cls): types = """\nimport * as ns2 from 'ns2';\n""" + ( ("\ndeclare module 'ns' " + cls._ns_spec_types) % cls._timestamp_def_formatted) + "\n\n" return types.replace('namespace', 'declare module') @classmethod def get_ns2_spec(cls): """Returns a simple namespace.""" return cls._ns2_spec @classmethod def get_ns2_types_as_declaration(cls): return (("\ndeclare module 'ns2' " + cls._ns2_spec_types ) % cls._timestamp_def_formatted) + "\n\n" @classmethod def get_all_types_as_namespace(cls): types = cls._error_types + "\n" + cls._timestamp_mapping + ";\n" + ( ("\nnamespace ns " + cls._ns_spec_types) % "") + ( ("\nnamespace ns2 " + cls._ns2_spec_types) % "") + "\n\n" return types @classmethod def get_ns3_spec_for_union(cls): """ Returns a test namespace which has a union field with all possible types of members a union can have. It includes (1) primitive, (2) struct, (3) enumerated subtypes, and (4) a union. This spec is useful in validating the auto-generated code for a union type defined in a namespace. """ return cls._ns3_union_spec @classmethod def get_ns3_types_as_declaration(cls): return (("\ndeclare module 'ns3' " + cls._ns3_union_spec_types ) % cls._timestamp_def_formatted) + "\n\n" class TestTSDTypesE2E(unittest.TestCase): def __init__(self, *args, **kwargs): super(TestTSDTypesE2E, self).__init__(*args, **kwargs) self.maxDiff = None # Increase text diff size def setUp(self): self.stone_output_directory = "output" if not os.path.exists(self.stone_output_directory): os.makedirs(self.stone_output_directory) self.template_file_name = "typescript.template" template_file_path = "{}/{}".format(self.stone_output_directory, self.template_file_name) with open(template_file_path, "w", encoding='utf-8') as template_file: template_file.write("/*TYPES*/") def tearDown(self): # Clear output of stone tool after all tests. shutil.rmtree('output') def _verify_generated_output(self, filename, expected_namespace_types): with open(filename, 'r', encoding='utf-8') as f: generated_types = f.read() self.assertEqual(generated_types, expected_namespace_types) def test_tsd_types_declarations_output(self): # Sanity check: stone must be importable for the compiler to work __import__('stone') # Compile spec by calling out to stone p = subprocess.Popen( [sys.executable, '-m', 'stone.cli', 'tsd_types', self.stone_output_directory, '--', self.template_file_name, '--exclude_error_types', '-i=0'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = p.communicate( input=(SpecHelper.get_ns_spec() + SpecHelper.get_ns2_spec()).encode('utf-8')) if p.wait() != 0: raise AssertionError('Could not execute stone tool: %s' % stderr.decode('utf-8')) # one file must be generated per namespace expected_ns_output = SpecHelper.get_ns_types_as_declaration() self._verify_generated_output('output/ns.d.ts', expected_ns_output) expected_ns2_output = SpecHelper.get_ns2_types_as_declaration() self._verify_generated_output('output/ns2.d.ts', expected_ns2_output) def test_tsd_types_namespace_output(self): # Sanity check: stone must be importable for the compiler to work __import__('stone') output_file_name = "all_types.ts" # Compile spec by calling out to stone p = subprocess.Popen( [sys.executable, '-m', 'stone.cli', 'tsd_types', self.stone_output_directory, '--', self.template_file_name, output_file_name, '-i=0'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = p.communicate( input=(SpecHelper.get_ns_spec() + SpecHelper.get_ns2_spec()).encode('utf-8')) if p.wait() != 0: raise AssertionError('Could not execute stone tool: %s' % stderr.decode('utf-8')) expected_output = SpecHelper.get_all_types_as_namespace() self._verify_generated_output('output/{}'.format(output_file_name), expected_output) def test_tsd_types_for_union(self): """ Test tsd types generated for a union which has all possible data types as members including primitive, struct, enumerated sub types and unions. """ # Sanity check: stone must be importable for the compiler to work __import__('stone') # Compile spec by calling out to stone p = subprocess.Popen( [sys.executable, '-m', 'stone.cli', 'tsd_types', self.stone_output_directory, '--', self.template_file_name, '--exclude_error_types', '-i=0'], stdin=subprocess.PIPE, stderr=subprocess.PIPE) _, stderr = p.communicate( input=(SpecHelper.get_ns3_spec_for_union()).encode('utf-8')) if p.wait() != 0: raise AssertionError('Could not execute stone tool: %s' % stderr.decode('utf-8')) # one file must be generated per namespace expected_ns_output = SpecHelper.get_ns3_types_as_declaration() self._verify_generated_output('output/ns3.d.ts', expected_ns_output) if __name__ == '__main__': unittest.main() stone-3.3.1/tox.ini000066400000000000000000000020041417406541500141640ustar00rootroot00000000000000[tox] envlist = py{35,36,37,38,py3},test_unit,check,lint,mypy,codecov skip_missing_interpreters = true [flake8] # See ignore = E128,E301,E302,E305,E402,W503,W504 max-line-length = 100 [testenv:test_unit] commands = pytest deps = -rtest/requirements.txt [testenv:check] commands = python setup.py bdist_wheel sdist twine check dist/* deps = setuptools wheel twine usedevelop = true [testenv:lint] commands = flake8 setup.py example stone test pylint --rcfile=.pylintrc setup.py example stone test deps = flake8 pylint # This probably breaks on Windows. See # . -rtest/requirements.txt usedevelop = true [testenv:mypy] commands = ./mypy-run.sh deps = enum34 mypy typed-ast usedevelop = true [testenv:codecov] commands = coverage run --rcfile=.coveragerc -m pytest coverage xml deps = -rtest/requirements.txt