pax_global_header00006660000000000000000000000064141213512600014505gustar00rootroot0000000000000052 comment=dbe8609e674d39e6e0f8e67c762b98dc795ea8be mathlib-tools-1.1.0/000077500000000000000000000000001412135126000142625ustar00rootroot00000000000000mathlib-tools-1.1.0/.github/000077500000000000000000000000001412135126000156225ustar00rootroot00000000000000mathlib-tools-1.1.0/.github/workflows/000077500000000000000000000000001412135126000176575ustar00rootroot00000000000000mathlib-tools-1.1.0/.github/workflows/ci.yml000066400000000000000000000026751412135126000210070ustar00rootroot00000000000000name: CI on: push: pull_request: release: types: [published] jobs: ci: runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: [macos-latest, ubuntu-latest, windows-latest] python-version: - name: 3.6 toxenv: py36 - name: 3.7 toxenv: py37 - name: 3.8 toxenv: py38 - name: 3.8 toxenv: mypy - name: 3.9 toxenv: py39 steps: - uses: actions/checkout@v2 - name: Set up Python ${{ matrix.python-version.name }} uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version.name }} - name: Ensure we have new enough versions to respect python_version run: python -m pip install -U pip - name: Install dependencies run: brew install gmp coreutils if: runner.os == 'macOS' - name: Install elan run: | curl https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh -sSf | sh -s -- -y - name: Add Lean to PATH run: echo "$HOME/.elan/bin/" >> $GITHUB_PATH if: runner.os != 'Windows' - name: Add Lean to PATH run: echo "${HOME}/.elan/bin" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append if: runner.os == 'Windows' - name: Install tox run: python -m pip install tox - name: Run tox run: python -m tox -e "${{ matrix.python-version.toxenv }}" mathlib-tools-1.1.0/.github/workflows/pyinstaller-windows.yml000066400000000000000000000013541412135126000244430ustar00rootroot00000000000000name: Package Application with Pyinstaller on: workflow_dispatch: push: branches: [ master ] pull_request: branches: [ master ] jobs: build: runs-on: windows-latest steps: - uses: actions/checkout@v2 - name: Set up Python uses: actions/setup-python@v2 with: python-version: 3.9 - name: Install pyinstaller run: python -m pip install pyinstaller -r pyinstaller-requirements.txt - name: Run pyinstaller run: python -m PyInstaller mathlibtools/leanproject.spec - name: Check the resulting exe functions run: ${{ github.workspace }}/dist/leanproject.exe --help - uses: actions/upload-artifact@v2 with: name: leanproject.exe path: dist/ mathlib-tools-1.1.0/.github/workflows/test_install_scripts.yml000066400000000000000000000006761412135126000246670ustar00rootroot00000000000000name: Install Scripts CI on: push: pull_request: release: types: [published] jobs: install_macos: runs-on: macos-latest steps: - uses: actions/checkout@v2 - name: Install mathlib-tools run: ./scripts/install_macos.sh - name: Check that lean runs run: lean --version - name: Check that leanproject runs run: leanproject --help - name: Check that VSCode exists run: code --version mathlib-tools-1.1.0/.github/workflows/test_nix.yml000066400000000000000000000004151412135126000222370ustar00rootroot00000000000000name: Test Nix Derivation on: [push, pull_request] jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v2 - uses: cachix/install-nix-action@v12 with: nix_path: nixpkgs=channel:nixos-20.09 - run: nix-build mathlib-tools-1.1.0/.gitignore000066400000000000000000000001601412135126000162470ustar00rootroot00000000000000*.olean /_target /leanpkg.path _cache __pycache__ *.pyc *.mypy_cache .python-version .tox build dist *.egg-info mathlib-tools-1.1.0/.vscode/000077500000000000000000000000001412135126000156235ustar00rootroot00000000000000mathlib-tools-1.1.0/.vscode/copyright.code-snippets000066400000000000000000000004171412135126000223340ustar00rootroot00000000000000{ "Copyright header for mathlib": { "scope": "lean", "prefix": "copyright", "body": [ "/-", "Copyright (c) ${CURRENT_YEAR} $1. All rights reserved.", "Released under Apache 2.0 license as described in the file LICENSE.", "Authors: $1", "-/" ] } }mathlib-tools-1.1.0/.vscode/settings.json000066400000000000000000000003641412135126000203610ustar00rootroot00000000000000{ "editor.insertSpaces": true, "editor.tabSize": 2, "editor.rulers" : [100], "files.encoding": "utf8", "files.eol": "\n", "files.insertFinalNewline": true, "files.trimFinalNewlines": true, "files.trimTrailingWhitespace": true } mathlib-tools-1.1.0/CHANGELOG.md000066400000000000000000000027451412135126000161030ustar00rootroot00000000000000# Change log ## 1.1.0 (2021-09-18) * Add `reduce-imports` command * Add `pull` command * `get-mathlib-cache` no longer understands `--rev`; if you want to use a different mathlib version, edit your `leanproject.toml`. If you are trying to get the cache when working on mathlib itself, use `get-cache --rev`. * Add `--fallback` to `get-cache` for traversing the git history to find an approximate cache. * `get-cache` no longer modifies `.lean` files in the working directory. * `mk-cache --force` no longer permits the working tree to be dirty. * `mk-all` now correctly handles filenames with special characters. ## 1.0.0 (2020-11-10) * Only look for .xz archives * Increase tolerance to weird git setups * Add pr command * Add rebase command * Add option --rev to get-cache and get-mathlib-cache * Drop python 3.5 support ## 0.0.10 (2020-07-28) * SSH handling tweaks ## 0.0.9 (2020-07-12) * Add mk-all command * Add decls command * Many small fixes ## 0.0.8 (2020-05-25) * Fix a bug and workaround some Windows bug ## 0.0.7 (2020-05-23) * Try to download .xz-compressed olean archives ## 0.0.6 (2020-05-09) * Add `leanproject get -b` to create a new branch ## 0.0.5 (2020-04-07) * Add import-graph command * Add delete-zombies command ## 0.0.4 (2020-03-24) * Add get-mathlib-cache command * Add --debug flag ## 0.0.3 (2020-03-11) Switch from update-mathlib to leanproject ## 0.0.2 (2019-12-28) Fix packaging issue ## 0.0.1 (2019-12-28) Version PyPi release of old mathlib tools mathlib-tools-1.1.0/CONTRIBUTING.md000066400000000000000000000042231412135126000165140ustar00rootroot00000000000000# Contributing to mathlib-tools Pull requests are welcome. The heaving lifting is done in `mathlibtools/lib.py` (which you can also use as a library for other python programs that want to manipulate Lean projects). Command line parsing is done in `mathlibtools/leanproject.py`, using the [click library](https://click.palletsprojects.com/en/7.x/). Please do not add code without type annotations. And of course you need to be able to run `mypy mathlibtools` without any error. ## Testing We run our tests using [pytest](https://docs.pytest.org/en/latest/). As usual with pytest, no much will work as expected if you don't install the package you want to test. So the first step is to run `pip install .` in the toplevel folder of this repository, the one containing `setup.py` (as usual, depending on your python setup, `pip` could be called `pip3`, and you may need administration permissions if you want to make a system-wide install). If you want to quickly modify code and retest, it is more convenient to use "editable install", by running `pip install -e .` which creates a link to your working copy instead of copying it (see [pip's documentation](https://pip.pypa.io/en/stable/reference/pip_install/#editable-installs) if needed). Once the package is installed, you can run `pytest` from the toplevel folder. Our continuous integration tests use python 3.6 to 3.9. This can be done locally using [tox](https://tox.readthedocs.io/en/latest/). For this you need `tox` of course, but also various versions of python. One convenient way to ensure that is to use [pyenv](https://github.com/pyenv/pyenv). After setting up `pyenv` and installing, say python 3.6.8, 3.7.2, 3.8.0 and 3.9.0 (using `pyenv install 3.6.8` etc.), you can create, inside the toplevel folder of your working copy, a file `.python-version` containing ``` 3.9.0 3.8.0 3.7.2 3.6.8 ``` Then you can run `tox` to run our test suite against all those versions of python. Tests in `tests/test_functional.py` are end-to-end tests that actually download things from the internet and write on disk (in temporary folders). They are pretty slow. Other test files are meant for unit tests. Don't hesitate to add tests! mathlib-tools-1.1.0/LICENSE000066400000000000000000000261351412135126000152760ustar00rootroot00000000000000 Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" form shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" form shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship, whether in Source or Object form, made available under the License, as indicated by a copyright notice that is included in or attached to the work (an example is provided in the Appendix below). "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and Derivative Works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display, publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or Object form. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE text file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright statement to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty, indemnity, or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or additional liability. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "{}" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in the appropriate comment syntax for the file format. We also recommend that a file or class name and description of purpose be included on the same "printed page" as the copyright notice for easier identification within third-party archives. Copyright {yyyy} {name of copyright owner} Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. mathlib-tools-1.1.0/README.md000066400000000000000000000045601412135126000155460ustar00rootroot00000000000000# mathlib-tools ![Test on Linux](https://github.com/leanprover-community/mathlib-tools/workflows/Test%20on%20Linux/badge.svg) ![Test on MacOS](https://github.com/leanprover-community/mathlib-tools/workflows/Test%20on%20MacOS/badge.svg) ![Test on Windows](https://github.com/leanprover-community/mathlib-tools/workflows/Test%20on%20Windows/badge.svg) This package contains `leanproject`, a supporting tool for [Lean mathlib](https://leanprover-community.github.io/). ## Installation In principle, you should install those tools as part of the [global Lean installation procedure](https://leanprover-community.github.io/get_started.html#regular-install) recommended by the Lean community. Read what what remains of this section only if you want more details about this specific part of the procedure (the tools described here won't give you anything if Lean itself is not available). ### Released version #### `pipx` The tools in this repository use python3, at least python 3.6, which is the oldest version of python supported by the python foundation. They can be installed using [pip](https://pypi.org/project/mathlibtools/). The basic install command for the latest released version is thus: ```bash python3 -m pip install mathlibtools ``` The above command may complain about permissions. This can be solved by running it as root, but this is not recommended in general. You can run `python3 -m pip install --user mathlibtools` to install it in your home directory (make sure that `$HOME/.local/bin/` is on your shell path afterwards), but an even better way is to use [pipx](https://pipxproject.github.io/pipx/): ```bash python3 -m pip install --user pipx python3 -m pipx ensurepath source ~/.profile pipx install mathlibtools ``` #### `macOS` If you are on macOS, the recommended way to install is via homebrew, which will handle the above Python installation for you: ```bash brew install mathlibtools ``` #### `NixOS` If you are using NixOS, you can also install mathlib tools using the bundled `default.nix` file: ``` nix-env -if https://github.com/leanprover-community/mathlib-tools/archive/master.tar.gz ``` ### Development version If you want to use the latest development version, you can clone this repository, go to the repository folder, and run `pip install .`. ## Usage See the [dedicated page](https://leanprover-community.github.io/leanproject.html) on the community website. mathlib-tools-1.1.0/default.nix000066400000000000000000000004351412135126000164300ustar00rootroot00000000000000{ pkgs ? import {} }: with pkgs.python3Packages; buildPythonApplication { pname = "mathlib-tools"; version = "1.0.0"; src = ./.; doCheck = false; propagatedBuildInputs = [ PyGithub GitPython toml click tqdm paramiko networkx pydot pyyaml atomicwrites ]; } mathlib-tools-1.1.0/mathlibtools/000077500000000000000000000000001412135126000167635ustar00rootroot00000000000000mathlib-tools-1.1.0/mathlibtools/__init__.py000066400000000000000000000000421412135126000210700ustar00rootroot00000000000000from ._version import __version__ mathlib-tools-1.1.0/mathlibtools/_version.py000066400000000000000000000004471412135126000211660ustar00rootroot00000000000000# Package versioning solution originally found here: # http://stackoverflow.com/q/458550 # Store the version here so: # 1) we don't load dependencies by storing it in __init__.py # 2) we can import it in setup.py for the same reason # 3) we can import it into your module __version__ = '1.1.0' mathlib-tools-1.1.0/mathlibtools/auth_github.py000066400000000000000000000016421412135126000216430ustar00rootroot00000000000000from git import Repo, InvalidGitRepositoryError # type: ignore from github import Github # type: ignore import configparser def auth_github(repo: Repo) -> Github: config = repo.config_reader() try: return Github(config.get('github', 'user'), config.get('github', 'password')) except configparser.NoSectionError: print('Info: No github section found in \'git config\', we will use GitHub with no authentication') return Github() except configparser.NoOptionError: try: return Github(config.get('github', 'oauthtoken')) except configparser.NoOptionError: print("Info: No github 'user'/'password' or 'oauthtoken' keys found in git config, " "we will use GitHub with no authentication.") print('You can create an OAuth token at https://github.com/settings/tokens/new (no scopes are required).') return Github() mathlib-tools-1.1.0/mathlibtools/decls.lean000066400000000000000000000026561412135126000207270ustar00rootroot00000000000000import data.list.sort meta.expr system.io open tactic declaration environment io io.fs (put_str_ln close) -- The next instance is there to prevent PyYAML trying to be too smart meta def my_name_to_string : has_to_string name := ⟨λ n, "\"" ++ to_string n ++ "\""⟩ local attribute [instance] my_name_to_string meta def pos_line (p : option pos) : string := match p with | some x := to_string x.line | _ := "" end meta def file_name (p : option string) : string := match p with | some x := x | _ := "" end meta def print_item_crawl (env : environment) (h : handle) (decl : declaration) : io unit := let name := decl.to_name in do put_str_ln h ((to_string name) ++ ":"), put_str_ln h (" File: " ++ file_name (env.decl_olean name)), put_str_ln h (" Line: " ++ pos_line (env.decl_pos name)) /-- itersplit l n will cut a list l into 2^n pieces (not preserving order) -/ meta def itersplit {α} : list α → ℕ → list (list α) | l 0 := [l] | l 1 := let (l1, l2) := l.split in [l1, l2] | l (k+2) := let (l1, l2) := l.split in itersplit l1 (k+1) ++ itersplit l2 (k+1) meta def main : io unit := do curr_env ← run_tactic get_env, h ← mk_file_handle "decls.yaml" mode.write, let decls := curr_env.fold [] list.cons, let filtered_decls := decls.filter (λ x, not (to_name x).is_internal), let pieces := itersplit filtered_decls 3, pieces.mmap' (λ l, l.mmap' (print_item_crawl curr_env h)), close h mathlib-tools-1.1.0/mathlibtools/delayed_interrupt.py000066400000000000000000000023321412135126000230600ustar00rootroot00000000000000import signal import logging # DelayedInterrupt class based on: # http://stackoverflow.com/a/21919644/487556 and # https://gist.github.com/tcwalther/ae058c64d5d9078a9f333913718bba95 class DelayedInterrupt(object): def __init__(self, signals): if not isinstance(signals, list) and not isinstance(signals, tuple): signals = [signals] self.sigs = signals def __enter__(self): self.signal_received = {} self.old_handlers = {} for sig in self.sigs: self.signal_received[sig] = False self.old_handlers[sig] = signal.getsignal(sig) def handler(s, frame): self.signal_received[sig] = (s, frame) # Note: in Python 3.5, you can use signal.Signals(sig).name logging.info('Signal %s received. Delaying KeyboardInterrupt.' % sig) self.old_handlers[sig] = signal.getsignal(sig) signal.signal(sig, handler) def __exit__(self, type, value, traceback): for sig in self.sigs: signal.signal(sig, self.old_handlers[sig]) if self.signal_received[sig] and self.old_handlers[sig]: self.old_handlers[sig](*self.signal_received[sig]) mathlib-tools-1.1.0/mathlibtools/git_helpers.py000066400000000000000000000060111412135126000216400ustar00rootroot00000000000000import git # type: ignore from typing import Callable, Iterator, Tuple, List def short_sha(rev: git.Commit) -> str: """ Truncate `rev.hexsha` without ambiguity """ return rev.repo.git.rev_parse(rev.hexsha, short=True) def visit_ancestors(rev: git.Commit) -> Iterator[Tuple[git.Commit, Callable]]: r""" Iterate over history, optionally pruning all ancestors of a given commit. This iterates backwards over history starting at `rev` and traversing the commit graph in topological (as opposed to date) order, ensuring that child commits are always visited before any of their parent commits. In this sense, this function is like ``repo.iter_commits(rev, topo_order=True)``. The key difference from ``iter_commits`` is that this version yields ``commit, prune`` pairs, where ``prune`` is a function accepting no arguments. If ``prune()`` is called, then the iterator will not visit any of the commits which are ancestors of ``commit``; that is, the history "tree" from that point backwards is pruned. As an example, consider a repository with the commit graph below, where ``A`` is the root commit and ``K`` and ``L`` are tips of branches:: A -- B -- E -- I -- J -- L \ / / C --- F -- H \ / D ---- G --- K The following code runs against this commit graph, and calls ``prune`` if it finds commits ``B``, ``F``, or ``G``:: >>> for c, prune in visit_ancestors(L): ... if c in {B, F, G}: ... prune() ... print('found ', c) ... else: ... print('visited', c) visited L visited J visited H visited I found G found F visited E As a result of calling ``prune()`` on commit ``G``, the ancestors of ``G`` (``D``, ``C``, ``B``, and ``A``) are pruned from the graph and never visited. The exact order that these commits appear in depends on the order of parents in merge commits, but since ``B`` is an ancestor of both ``F`` and ``G``, it will always be pruned before it is visited. """ repo = rev.repo pruned_commits : List[git.Commit] = [] # the commits to ignore along with their ancestors skip_n = 0 # the index to resume the iteration while True: args = [rev] + ['--not'] + pruned_commits proc = repo.git.rev_list(*args, as_process=True, skip=skip_n, topo_order=True) for c in git.Commit._iter_from_process_or_stream(repo, proc): # build a temporary function to hand back to the user do_prune = False def prune(): nonlocal do_prune do_prune = True yield c, prune if do_prune: pruned_commits.append(c) break else: # start after this commit next time we restart the search skip_n += 1 else: # all ancestors found return mathlib-tools-1.1.0/mathlibtools/import_graph.py000066400000000000000000000052211412135126000220300ustar00rootroot00000000000000from pathlib import Path from typing import Optional import tempfile import subprocess import networkx as nx # type: ignore class ImportGraph(nx.DiGraph): def __init__(self, base_path: Optional[Path] = None) -> None: """A Lean project import graph.""" super().__init__(self) self.base_path = base_path or Path('.') def to_dot(self, path: Optional[Path] = None) -> None: """Writes itself to a graphviz dot file.""" path = path or self.base_path/'import_graph.dot' nx.drawing.nx_pydot.to_pydot(self).write_dot(str(path)) def to_gexf(self, path: Optional[Path] = None) -> None: """Writes itself to a gexf dot file, suitable for Gephi.""" path = path or self.base_path/'import_graph.gexf' nx.write_gexf(self, str(path)) def to_graphml(self, path: Optional[Path] = None) -> None: """Writes itself to a gexf dot file, suitable for yEd.""" path = path or self.base_path/'import_graph.graphml' nx.write_graphml(self, str(path)) def write(self, path: Path): if path.suffix == '.dot': self.to_dot(path) elif path.suffix == '.gexf': self.to_gexf(path) elif path.suffix == '.graphml': self.to_graphml(path) elif path.suffix in ['.pdf', '.svg', '.png']: dot_format = '-T' + path.suffix[1:] with tempfile.TemporaryDirectory() as tmpdirname: tmpf = Path(tmpdirname)/'tmp.dot' self.to_dot(tmpf) with path.open('w') as outf: subprocess.run(['dot', dot_format, str(tmpf)], stdout=outf) else: raise ValueError('Unsupported graph output format. ' 'Use .dot, .gexf, .graphml or a valid ' 'graphviz output format (eg. .pdf).') def ancestors(self, node: str) -> 'ImportGraph': """Returns the subgraph leading to node.""" H = self.subgraph(nx.ancestors(self, node).union([node])) H.base_path = self.base_path return H def descendants(self, node: str) -> 'ImportGraph': """Returns the subgraph descending from node.""" H = self.subgraph(nx.descendants(self, node).union([node])) H.base_path = self.base_path return H def path(self, start: str, end: str) -> 'ImportGraph': """Returns the subgraph descending from the start node and used by the end node.""" D = self.descendants(start) A = self.ancestors(end) H = self.subgraph(set(D.nodes).intersection(A.nodes)) H.base_path = self.base_path return H mathlib-tools-1.1.0/mathlibtools/leanproject.py000066400000000000000000000340271412135126000216510ustar00rootroot00000000000000import sys from pathlib import Path from datetime import datetime from typing import Tuple, Optional from getpass import getpass from git.exc import GitCommandError # type: ignore import click from mathlibtools.lib import (LeanProject, log, InvalidLeanProject, LeanDownloadError, set_download_url, touch_oleans, CacheFallback) # Click aliases from Stephen Rauch at # https://stackoverflow.com/questions/46641928 class CustomMultiCommand(click.Group): def command(self, *args, **kwargs): """Behaves the same as `click.Group.command()` except if passed a list of names, all after the first will be aliases for the first. """ def decorator(f): if args and isinstance(args[0], list): _args = [args[0][0]] + list(args[1:]) for alias in args[0][1:]: cmd = super(CustomMultiCommand, self).command( alias, *args[1:], **kwargs)(f) cmd.short_help = "Alias for '{}'".format(_args[0]) cmd.hidden = True else: _args = args cmd = super(CustomMultiCommand, self).command( *_args, **kwargs)(f) return cmd return decorator """Allows the user to shorten commands to a (unique) prefix.""" def get_command(self, ctx, cmd_name): rv = click.Group.get_command(self, ctx, cmd_name) if rv is not None: return rv matches = [x for x in self.list_commands(ctx) if x.startswith(cmd_name)] if not matches: return None elif len(matches) == 1: return click.Group.get_command(self, ctx, matches[0]) ctx.fail('Too many matches: %s' % ', '.join(sorted(matches))) def proj() -> LeanProject: return LeanProject.from_path(Path('.'), cache_url, force_download, lean_upgrade) # The following are global state variables. This is a lazy way of propagating # the global options. cache_url = '' force_download = False lean_upgrade = True debug = False def handle_exception(exc, msg): if debug: raise exc else: log.error(msg) sys.exit(-1) @click.group(cls=CustomMultiCommand, context_settings={ 'help_option_names':['-h', '--help']}) @click.option('--from-url', '-u', default='', nargs=1, help='Override base url for olean cache.') @click.option('--force-download', '-f', 'force', default=False, is_flag=True, help='Download olean cache without looking for a local version.') @click.option('--no-lean-upgrade', 'noleanup', default=False, is_flag=True, help='Do not upgrade Lean version when upgrading mathlib.') @click.option('--debug', 'python_debug', default=False, is_flag=True, help='Display python tracebacks in case of error.') @click.version_option() def cli(from_url: str, force: bool, noleanup: bool, python_debug: bool) -> None: """Command line client to manage Lean projects depending on mathlib. Use leanproject COMMAND --help to get more help on any specific command.""" global cache_url, force_download, lean_upgrade, debug cache_url = from_url force_download = force lean_upgrade = not noleanup debug = python_debug @cli.command() @click.argument('path', default='.') def new(path: str = '.') -> None: """Create a new Lean project and prepare mathlib. If no directory name is given, the current directory is used. """ LeanProject.new(Path(path), cache_url, force_download) @cli.command() def add_mathlib() -> None: """Add mathlib to the current project.""" proj().add_mathlib() @cli.command(['upgrade-mathlib', 'update-mathlib', 'up']) def upgrade_mathlib() -> None: """Upgrade mathlib (as a dependency or as the main project).""" try: proj().upgrade_mathlib() except LeanDownloadError as err: handle_exception(err, 'Failed to fetch mathlib oleans') except InvalidLeanProject: project = LeanProject.user_wide(cache_url, force_download) project.upgrade_mathlib() @cli.command() def build() -> None: """Build the current project.""" proj().build() def parse_project_name(name: str, ssh: bool = True) -> Tuple[str, str, str, bool]: """Parse the name argument for get_project Returns (name, url, branch, is_url). If name is not a full url, the returned url will be a https or ssh url depending on the boolean argument ssh. """ # This is split off the actual command function for # unit testing purposes if ':' in name: pieces = name.split(':') if len(pieces) >= 3: name = ':'.join(pieces[:-1]) branch = pieces[-1] elif 'http' in pieces[0] or '@' in pieces[0]: branch = '' else: name, branch = pieces else: branch = '' if not name.startswith(('git@', 'http')): if '/' not in name: org_name = 'leanprover-community/'+name else: org_name, name = name, name.split('/')[1] if ssh: url = 'git@github.com:'+org_name+'.git' else: url = 'https://github.com/'+org_name+'.git' is_url = False else: url = name name = name.split('/')[-1].replace('.git', '') is_url = True return name, url, branch, is_url @cli.command(name='get') @click.argument('name') @click.argument('directory', default='') @click.option('--new-branch', '-b', default=False, is_flag=True, help='Create a new branch.') def get_project(name: str, new_branch: bool, directory: str = '') -> None: """Clone a project from a GitHub name or git url. Put it in dir if this argument is given. A GitHub name without / will be considered as a leanprover-community project. If the name ends with ':foo' then foo will be interpreted as a branch name, and that branch will be checked out. This will fail if the branch does not exist. If you want to create a new branch, pass the `-b` option.""" original_name = name name, url, branch, is_url = parse_project_name(original_name) if branch: name = name + '_' + branch directory = directory or name if directory and Path(directory).exists(): raise FileExistsError('Directory ' + directory + ' already exists') try: LeanProject.from_git_url(url, directory, branch, new_branch, cache_url, force_download) except GitCommandError as err: # if full url is provided, do not retry with HTTPS if not is_url: log.info('Error cloning via SSH, trying HTTPS...') try: name, url, branch, is_url = parse_project_name(original_name, ssh=False) LeanProject.from_git_url(url, directory, branch, new_branch, cache_url, force_download) except GitCommandError as e: handle_exception(e, e.stderr) else: handle_exception(err, err.stderr) @cli.command() @click.option('--force', default=False, is_flag=True, help='Make cache even if the cache already exists.') def mk_cache(force: bool = False) -> None: """Cache olean files. The repository must be clean in order to ensure there is a suitable git commit to associate the hash with.""" proj().mk_cache(force) @cli.command() @click.option('--rev', default=None, help='A git sha.') @click.option('--fallback', type=click.Choice(['none', 'show', 'download-first', 'download-all']), default='show', help="Behavior if no matching cache is available.") def get_cache(rev: Optional[str], fallback: str) -> None: """Restore olean files from a cache. \b The fallback parameter is interpreted as follows: none: fail without trying anything else show: show but do not download possible fallback caches download-first: show all fallback caches, download and apply the first download-all: show and download all fallback caches, apply the first. """ fallback_enum = CacheFallback(fallback) try: proj().get_cache(rev, fallback_enum) except (LeanDownloadError, FileNotFoundError) as err: handle_exception(err, 'Failed to fetch cached oleans') @cli.command() def get_mathlib_cache() -> None: """Get mathlib .lean and .olean files in a project depending on mathlib, without upgrading.""" project = proj() try: project.get_mathlib_olean() except (LeanDownloadError, FileNotFoundError) as err: handle_exception(err, 'Failed to fetch mathlib oleans') @cli.command() def delete_zombies() -> None: """Delete zombie oleans, .olean files with no matching .lean files""" proj().delete_zombies() @cli.command() def clean() -> None: """Delete all olean files""" proj().clean() @cli.command() def hooks() -> None: """Setup git hooks for the current project.""" proj().setup_git_hooks() @cli.command() @click.argument('url') def set_url(url: str) -> None: """Set the default url where oleans should be fetched.""" set_download_url(url) @cli.command() def check() -> None: """Check mathlib oleans are more recent than their sources""" project = proj() core_ok, mathlib_ok = project.check_timestamps() toolchain = project.toolchain toolchain_path = Path.home()/'.elan'/'toolchains'/toolchain if not core_ok: print('Some core oleans files in toolchain {} seem older than ' 'their source.'.format(toolchain)) touch = input('Do you want to set their modification time to now (y/n) ? ') if touch.lower() in ['y', 'yes']: touch_oleans(toolchain_path) if not mathlib_ok: print('Some mathlib oleans files seem older than their source.') touch = input('Do you want to set their modification time to now (y/n) ? ') if touch.lower() in ['y', 'yes']: touch_oleans(project.mathlib_folder/'src') if core_ok and mathlib_ok: log.info('Everything looks fine.') @cli.command() def mk_all() -> None: """Creates all.lean importing everything from the project.""" proj().make_all() @cli.command() def global_install() -> None: """Install mathlib user-wide.""" proj = LeanProject.user_wide(cache_url, force_download) proj.add_mathlib() @cli.command() def global_upgrade() -> None: """Upgrade user-wide mathlib""" proj = LeanProject.user_wide(cache_url, force_download) proj.upgrade_mathlib() @cli.command() @click.option('--to', 'to', default=None, help='Return only imports leading to this file.') @click.option('--from', 'from_', default=None, help='Return only imports starting from this file.') @click.argument('output', default='import_graph.dot') def import_graph(to: Optional[str], from_: Optional[str], output: str) -> None: """Write an import graph for this project. Arguments for '--to' and '--from' should be specified as Lean imports (e.g. 'data.mv_polynomial') rather than file names. You may specify an output filename, and the suffix will determine the output format. By default the graph will be written to 'import_graph.dot'. For .dot, .pdf, .svg, or .png output you will need to install 'graphviz' first. """ project = proj() graph = project.import_graph if to and from_: G = graph.path(start=from_, end=to) elif to: G = graph.ancestors(to) elif from_: G = graph.descendants(from_) else: G = graph G.write(Path(output)) @cli.command() @click.option('--sed', 'sed', default=False, is_flag=True, help='Instead of printing a list of removable imports, print a sed script that can be run to remove the imports.') @click.argument('file', default=None, required=False) def reduce_imports(file: str, sed: bool = False) -> None: """List imports that can be removed in the project in the format `("source.file", ["removable.import", "another.removable.import"])`. Argument '--file' should be specified as a Lean import (e.g. 'data.mv_polynomial') rather than a file name. """ project = proj() if sed: print("# on mac use gsed instead of sed") for l in project.reduce_imports_sed(file=file): print(l) else: for t in project.reduce_imports(file=file): print(t) @cli.command() @click.argument('path', default='') def decls(path: str = '') -> None: """List declarations seen from this project If no file name is given, the result will be in decls.yaml in the project root. """ project = proj() decls = project.list_decls() outpath = Path(path) if path else project.directory/'decls.yaml' with outpath.open('w') as outfile: for name, info in decls.items(): outfile.write('{}:\n origin: {}\n path: {}\n line: {}\n'.format( name, info.origin, info.filepath, info.line)) @cli.command() @click.argument('branch_name') @click.option('--force', default=False, is_flag=True, help='Update master and create branch even if the repository is dirty.') def pr(branch_name: str, force: bool = False) -> None: """Prepare to work on a mathlib pull-request on a new branch.""" proj().pr(branch_name, force) @cli.command() @click.option('--force', default=False, is_flag=True, help='Rebase on master even if the repository is dirty.') def rebase(force: bool = False) -> None: """ On mathlib, update master, get oleans and rebase current branch. """ proj().rebase(force) @cli.command() @click.argument('remote', default='origin') def pull(remote: str = '') -> None: """ Pull and get mathlib oleans. Default remote is 'origin'. """ proj().pull(remote) def safe_cli(): try: cli() # pylint: disable=no-value-for-parameter except Exception as err: handle_exception(err, str(err)) if __name__ == "__main__": # This allows `python3 -m mathlibtools.leanproject`. # This is useful for when python is on the path but its installed scripts are not # It also allows pyinstaller to create leanproject.exe standalone executable for Windows safe_cli() mathlib-tools-1.1.0/mathlibtools/leanproject.spec000066400000000000000000000020721412135126000221460ustar00rootroot00000000000000# -*- mode: python ; coding: utf-8 -*- """ Originally generated with `pyinstaller`, with the command `pyinstaller leanproject.py`. In future, we may want to use the approach described at https://github.com/pyinstaller/pyinstaller/wiki/Recipe-Setuptools-Entry-Point, if we end up adding more entrypoints. """ block_cipher = None a = Analysis(['leanproject.py'], binaries=[], datas=[], hiddenimports=[], hookspath=[], runtime_hooks=[], excludes=[], win_no_prefer_redirects=False, win_private_assemblies=False, cipher=block_cipher, noarchive=False) pyz = PYZ(a.pure, a.zipped_data, cipher=block_cipher) exe = EXE(pyz, a.scripts, a.binaries, a.zipfiles, a.datas, [], name='leanproject', debug=False, bootloader_ignore_signals=False, strip=False, upx=True, upx_exclude=[], runtime_tmpdir=None, console=True ) mathlib-tools-1.1.0/mathlibtools/lib.py000066400000000000000000001157661412135126000201230ustar00rootroot00000000000000from pathlib import Path import logging import tempfile import shutil import tarfile import signal import re import os import stat import platform import subprocess import pickle import contextlib import enum from datetime import datetime import concurrent.futures import tarfile from typing import Iterable, Union, List, Tuple, Optional, Dict, TYPE_CHECKING from tempfile import TemporaryDirectory import shutil import requests from tqdm import tqdm # type: ignore import toml import yaml from git import (Repo, Commit, InvalidGitRepositoryError, # type: ignore GitCommandError, BadName, RemoteReference) # type: ignore from atomicwrites import atomic_write if TYPE_CHECKING: from mathlibtools.import_graph import ImportGraph from mathlibtools.delayed_interrupt import DelayedInterrupt from mathlibtools.auth_github import auth_github, Github from mathlibtools.git_helpers import visit_ancestors, short_sha log = logging.getLogger("Mathlib tools") log.setLevel(logging.INFO) if (log.hasHandlers()): log.handlers.clear() log.addHandler(logging.StreamHandler()) class InvalidLeanProject(Exception): pass class InvalidMathlibProject(Exception): """A mathlib project is a Lean project depending on mathlib""" pass class LeanDownloadError(Exception): pass class LeanDirtyRepo(Exception): pass class InvalidLeanVersion(Exception): pass class LeanProjectError(Exception): pass DOT_MATHLIB = Path(os.environ.get("MATHLIB_CACHE_DIR") or Path.home()/'.mathlib') AZURE_URL = 'https://oleanstorage.azureedge.net/mathlib/' DOT_MATHLIB.mkdir(parents=True, exist_ok=True) DOWNLOAD_URL_FILE = DOT_MATHLIB/'url' MATHLIB_URL = 'https://github.com/leanprover-community/mathlib.git' LEAN_VERSION_RE = re.compile(r'(.*)\t.*refs/heads/lean-(.*)') # This regex is from [1] and implements the logic at [2]. # [1]: https://github.com/leanprover/vscode-lean/blob/2b43982c4c6305a0f20f156152a60613a6f1a683/syntaxes/lean.json#L193 # [2]: https://github.com/leanprover-community/lean/blob/65ad4ffdb3abac75be748554e3cbe990fb1c6500/src/util/name.cpp#L65-L83 LEAN_UNESCAPED_IDENTIFIER_RE = re.compile( r"(?![λΠΣ])[_a-zA-Zα-ωΑ-Ωϊ-ϻἀ-῾℀-⅏𝒜-𝖟](?:(?![λΠΣ])[_a-zA-Zα-ωΑ-Ωϊ-ϻἀ-῾℀-⅏𝒜-𝖟0-9'ⁿ-₉ₐ-ₜᵢ-ᵪ])*") VersionTuple = Tuple[int, int, int] def mathlib_lean_version() -> VersionTuple: """Return the latest Lean release supported by mathlib""" resp = requests.get("https://raw.githubusercontent.com/leanprover-community/mathlib/master/leanpkg.toml") assert resp.status_code == 200 conf = toml.loads(resp.text) return parse_version(conf['package']['lean_version']) def set_download_url(url: str = AZURE_URL) -> None: """Store the download url in .mathlib.""" DOWNLOAD_URL_FILE.write_text(url) def get_download_url() -> str: """Get the download url from .mathlib.""" return DOWNLOAD_URL_FILE.read_text().strip('/\n')+'/' if not DOWNLOAD_URL_FILE.exists(): set_download_url() def pack(root: Path, srcs: Iterable[Path], target: Path) -> None: """Creates, as target, a tar.xz archive containing all paths from src, relative to the folder root""" try: target.unlink() except FileNotFoundError: pass cur_dir = Path.cwd() with DelayedInterrupt([signal.SIGTERM, signal.SIGINT]): os.chdir(str(root)) ar = tarfile.open(str(target), 'w|xz') for src in srcs: ar.add(str(src.relative_to(root))) ar.close() os.chdir(str(cur_dir)) def unpack_archive(fname: Union[str, Path], tgt_dir: Union[str, Path], oleans_only: bool) -> None: """ Alternative to `shutil.unpack_archive` that shows progress""" with tarfile.open(fname) as tarobj: if oleans_only: members : Iterable[tarfile.TarInfo] = (f for f in tarobj if Path(f.name).suffix == '.olean') else: members = tarobj tarobj.extractall( str(tgt_dir), members=tqdm(members, desc=' files extracted', unit='')) def escape_identifier(s : str) -> str: """ Helper function to wrap _pieces_ of identifiers in double french quotes if they need to be wrapped by lean, we use this for file paths so we also escape strings of the form `a.a` even though they are valid identifiers. By escaping strings we ensure that lean accepts them as imports""" if re.fullmatch(LEAN_UNESCAPED_IDENTIFIER_RE, s): return s return "«" + s + "»" class OleanCache: """ A reference to a cache of oleans for a single commit. This is a context manager so that references to caches which hold onto resources can clean up when those resources are no longer needed. """ def __init__(self, locator: 'CacheLocator', rev: Commit): self.locator = locator self.rev = rev self.path = self.locator.cache_dir / self.fname @property def fname(self) -> str: return self.rev.hexsha + '.tar.xz' def make_local(self) -> 'LocalOleanCache': raise NotImplementedError def close(self) -> None: pass def __enter__(self) -> 'OleanCache': return self def __exit__(self, *args) -> None: self.close() class LocalOleanCache(OleanCache): """ A cache of oleans that lives on the local filesystem. Any cache can be converted into a local cache via `OleanCache.download`.""" def __init__(self, locator: 'CacheLocator', rev): super().__init__(locator, rev) if not self.path.exists(): raise LookupError("Local cache not found") def make_local(self) -> 'LocalOleanCache': return self # already downloaded class RemoteOleanCache(OleanCache): """ A cache of oleans that lives on a remove server. This holds an open HTTP connection to the server from which the cache can be downloaded.""" def __init__(self, locator: 'CacheLocator', rev): super().__init__(locator, rev) assert self.locator.cache_url is not None self.req = requests.get(self.locator.cache_url + self.fname, stream=True) self.req.raise_for_status() def close(self): self.req.close() def make_local(self): # download the cache atomically from the already-open connection with atomic_write(self.path, mode='wb', overwrite=True) as tgt: total_size = int(self.req.headers.get('content-length', 0)) with tqdm.wrapattr(self.req.raw, "read", total=total_size, desc=' ' + short_sha(self.rev)) as src: shutil.copyfileobj(src, tgt) self.req.close() return LocalOleanCache(self.locator, self.rev) class CacheFallback(enum.Enum): """ Specifies the fallback to use when an exactly matching cache is not available """ NONE = 'none' DOWNLOAD_FIRST = 'download-first' DOWNLOAD_ALL = 'download-all' SHOW = 'show' class CacheLocator: """ A helper class to locate and download caches for a given repo and remote URL """ def __init__(self, name: str, repo: Repo, cache_url: Optional[str], cache_dir: Path, *, force_download=False): self.name = name self.repo = repo self.cache_url = cache_url self.cache_dir = cache_dir self.force_download = force_download def find_exact(self, rev: Commit) -> Optional[OleanCache]: """ Find a cache that is for `rev` exactly """ log.info(f"Looking for {self.name} oleans for {short_sha(rev)}") if not self.force_download: log.info(f' locally...') try: local_c = LocalOleanCache(self, rev) except LookupError: pass else: log.info(f' Found local {self.name} oleans') return local_c if self.cache_url is not None: log.info(' remotely...') try: remote_c = RemoteOleanCache(self, rev) except requests.HTTPError: pass else: log.info(f' Found remote {self.name} oleans') return remote_c return None def find_all(self, rev: Commit) -> Tuple[contextlib.ExitStack, List[OleanCache]]: """ Find all closest ancestors that have a cache. Returns a tuple where the first result is a contextmanager that will close any unused http requests """ caches = [] with contextlib.ExitStack() as stack: for parent_commit, prune in visit_ancestors(rev): cache = self.find_exact(parent_commit) if cache is None: log.info(f"No cache available for revision {short_sha(parent_commit)}") else: stack.enter_context(cache) # ensure we do not leak requests caches.append(cache) prune() # do not visit the ancestors of this commit return stack.pop_all(), caches # https://github.com/python/mypy/issues/7726 assert False def find_local_with_fallback(self, rev: Commit, fallback: CacheFallback) -> LocalOleanCache: """ Find (or download) a local cache for `rev` using the provided fallback strategy. """ # if fallback is `NONE`, do not even attempt a search (to conserve network access) if fallback == CacheFallback.NONE: cache = self.find_exact(rev) if not cache: raise LeanDownloadError(f"No cache was available for {short_sha(rev)}.\n") with cache: log.info("Located matching cache") return cache.make_local() # Otherwise, do a search. This will open as many HTTP connections as # necessary, which the `with` statement cleans up. ctx, caches = self.find_all(rev) with ctx: if not caches: # this should never happen unless azure goes down raise LeanProjectError('No archives available for any commits!') cache = caches[0] if cache.rev == rev: assert len(caches) == 1 log.info("Using matching cache") return caches[0].make_local() if len(caches) > 1: archive_items = ''.join([f'\n * {short_sha(c.rev)}' for c in caches]) commit_args = ''.join([f' {short_sha(c.rev)}^!' for c in caches]) log.warn( f"No cache was available for {short_sha(rev)}.\n" f"There are multiple viable caches from parent commits:{archive_items}\n" f"To see the commits in question, run:\n" f" git log --graph {short_sha(rev)}{commit_args}") else: log.warn( f"No cache was available for {short_sha(rev)}. " f"A cache was found for the ancestor {short_sha(cache.rev)}.\n" f"To see the intermediate commits, run:\n" f" git log --graph {short_sha(rev)} {short_sha(cache.rev)}^!") if fallback == CacheFallback.DOWNLOAD_ALL: log.info("Preparing all caches, using the first") with concurrent.futures.ThreadPoolExecutor() as executor: local_caches = list(executor.map(lambda c: c.make_local(), caches)) return local_caches[0] elif fallback == CacheFallback.DOWNLOAD_FIRST: log.info("Using first cache") return caches[0].make_local() elif fallback == CacheFallback.SHOW: log.info(f"Run `leanproject get-cache --rev` on one of the available commits above.") raise LeanDownloadError else: raise RuntimeError('Invalid fallback argument') # https://github.com/python/mypy/issues/7726 assert False def parse_version(version: str) -> VersionTuple: """Turn a lean version string into a tuple of integers or raise InvalidLeanVersion""" #Something that could be in a branch name or modern leanpkg.toml m = re.match(r'.*lean[-:](.*)\.(.*)\.(.*).*', version) if m: return (int(m.group(1)), int(m.group(2)), int(m.group(3))) # The output of `lean -- version` m = re.match(r'.*version (.*)\.(.*)\.(.*),.*', version) if m: return (int(m.group(1)), int(m.group(2)), int(m.group(3))) # Only a version string m = re.match(r'(.*)\.(.*)\.(.*)', version) if m: return (int(m.group(1)), int(m.group(2)), int(m.group(3))) raise InvalidLeanVersion(version) def lean_version_toml(version: VersionTuple) -> str: """Turn a Lean version tuple into the string expected in leanpkg.toml.""" ver_str = '{:d}.{:d}.{:d}'.format(*version) if version < (3, 5, 0): return ver_str else: return 'leanprover-community/lean:' + ver_str def clean(dir: Path) -> None: log.info('cleaning {} ...'.format(str(dir))) for path in dir.glob('**/*.olean'): path.unlink() def delete_zombies(dir: Path) -> None: for path in dir.glob('**/*.olean'): if not path.with_suffix('.lean').exists(): log.info('deleting zombie {} ...'.format(str(path))) path.unlink() def check_core_timestamps(toolchain: str) -> bool: """Check that oleans are more recent than their source in core lib""" toolchain_path = Path.home()/'.elan'/'toolchains'/toolchain try: return all(p.stat().st_mtime < p.with_suffix('.olean').stat().st_mtime for p in toolchain_path.glob('**/*.lean')) except FileNotFoundError: return False def touch_oleans(path: Path) -> None: """Set modification time for oleans in path and its subfolders to now""" now = datetime.now().timestamp() for p in path.glob('**/*.olean'): os.utime(str(p), (now, now)) def find_root(path: Path) -> Path: """ Find a Lean project root in path by searching for leanpkg.toml in path and its ancestors. """ if (path/'leanpkg.toml').exists(): return path parent = path.absolute().parent if parent != path: return find_root(parent) else: raise InvalidLeanProject('Could not find a leanpkg.toml') class DeclInfo: def __init__(self, origin: str, filepath: Path, line: int): """Implementation information for a declaration. The origin argument is meant to be either 'core' or a project name, including mathlib.""" self.origin = origin self.filepath = filepath self.line = line def __repr__(self) -> str: return "DeclInfo('{}', '{}', {})".format(self.origin, self.filepath, self.line) class LeanProject: def __init__(self, repo: Optional[Repo], is_dirty: bool, rev: str, directory: Path, pkg_config: dict, deps: dict, cache_url: str = '', force_download: bool = False, upgrade_lean: bool = True) -> None: """A Lean project.""" self.repo = repo self.is_dirty = is_dirty self.rev = rev self.directory = directory.absolute().resolve() self.pkg_config = pkg_config self.src_directory = self.directory/pkg_config.get('path', '') self.deps = deps self.cache_url = cache_url or get_download_url() self.force_download = force_download self.upgrade_lean = upgrade_lean self._import_graph = None # type: Optional[ImportGraph] @classmethod def from_path(cls, path: Path, cache_url: str = '', force_download: bool = False, upgrade_lean: bool = True) -> 'LeanProject': """Builds a LeanProject from a Path object""" try: repo = Repo(path, search_parent_directories=True) except InvalidGitRepositoryError: repo = None is_dirty = False rev = '' if repo: if repo.bare: raise InvalidLeanProject('Git repository is not initialized') is_dirty = repo.is_dirty() try: rev = repo.commit().hexsha except ValueError: rev = '' directory = find_root(path) config = toml.load(directory/'leanpkg.toml') return cls(repo, is_dirty, rev, directory, config['package'], config['dependencies'], cache_url, force_download, upgrade_lean) @classmethod def user_wide(cls, cache_url: str = '', force_download: bool = False) -> 'LeanProject': """Return the user-wide LeanProject (living in ~/.lean) If the project does not exist, it will be created, using the latest version of Lean supported by mathlib.""" directory = Path.home()/'.lean' try: config = toml.load(directory/'leanpkg.toml') except FileNotFoundError: directory.mkdir(exist_ok=True) version = mathlib_lean_version() if version <= (3, 4, 2): version_str = '.'.join(map(str, version)) else: version_str = 'leanprover-community/lean:' +\ '.'.join(map(str, version)) pkg = { 'name': '_user_local_packages', 'version': '1', 'lean_version': version_str } with (directory/'leanpkg.toml').open('w') as pkgtoml: toml.dump({'package': pkg}, pkgtoml) config = { 'package': pkg, 'dependencies': dict() } return cls(None, False, '', directory, config['package'], config['dependencies'], cache_url, force_download) @property def name(self) -> str: return self.pkg_config['name'] @property def lean_version(self) -> VersionTuple: return parse_version(self.pkg_config['lean_version']) @lean_version.setter def lean_version(self, version: VersionTuple) -> None: self.pkg_config['lean_version'] = lean_version_toml(version) @property def is_mathlib(self) -> bool: return self.name == 'mathlib' @property def toolchain(self) -> str: ver_str = '{:d}.{:d}.{:d}'.format(*self.lean_version) return ver_str if self.lean_version < (3, 5, 0) \ else 'leanprover-community-lean-' + ver_str @property def mathlib_rev(self) -> str: if self.is_mathlib: return self.rev if 'mathlib' not in self.deps: raise InvalidMathlibProject('This project does not depend on mathlib') try: rev = self.deps['mathlib']['rev'] except KeyError: raise InvalidMathlibProject( 'Project seems to refer to a local copy of mathlib ' 'instead of a GitHub repository') return rev @property def mathlib_folder(self) -> Path: if self.is_mathlib: return self.directory else: return self.directory/'_target'/'deps'/'mathlib' @property def mathlib_repo(self) -> Repo: if self.is_mathlib: assert self.repo return self.repo else: if not self.mathlib_folder.exists(): self.run_echo(['leanpkg', 'configure']) return Repo(self.mathlib_folder) def read_config(self) -> None: try: config = toml.load(self.directory/'leanpkg.toml') except FileNotFoundError: raise InvalidLeanProject('Missing leanpkg.toml') self.deps = config['dependencies'] self.pkg_config = config['package'] def write_config(self) -> None: """Write leanpkg.toml for this project.""" # Fix leanpkg lean_version bug if needed (the lean_version property # setter is working here, hence the weird line). self.lean_version = self.lean_version # Note we can't blindly use toml.dump because we need dict as values # for dependencies. with (self.directory/'leanpkg.toml').open('w') as cfg: cfg.write('[package]\n') cfg.write(toml.dumps(self.pkg_config)) cfg.write('\n[dependencies]\n') for dep, val in self.deps.items(): nval = str(val).replace("'git':", 'git =').replace( "'rev':", 'rev =').replace("'", '"') cfg.write('{} = {}\n'.format(dep, nval)) def get_mathlib_olean(self) -> None: """Get precompiled mathlib oleans for this project (which depends on mathlib)""" if self.is_mathlib: # user should have run `get-cache` not `get-mathlib-cache log.warn("`get-mathlib-cache` is for projects which depend on " "mathlib, not for mathlib itself. " "Running `get-cache` instead.") return self.get_cache() repo = self.mathlib_repo try: commit = repo.rev_parse(self.mathlib_rev) except BadName: # presumably the mathlib folder is outdated log.info("Can't find the required mathlib revision, will try to update" "mathlib git repository") self.run_echo(['leanpkg', 'configure']) commit = repo.rev_parse(self.mathlib_rev) # Just in case the user broke the workflow (for instance git clone # mathlib by hand and then run `leanproject get-cache`) if not (self.directory/'leanpkg.path').exists(): self.run(['leanpkg', 'configure']) cache_locator = CacheLocator(self.name, repo, self.cache_url, DOT_MATHLIB, force_download=self.force_download) # We want an exact match here; if we can't find one, then the user should # just point their config file at a version of mathlib with a cache. cache = cache_locator.find_local_with_fallback(commit, fallback=CacheFallback.NONE) log.info("Applying cache") self.clean_mathlib_dep() self.mathlib_folder.mkdir(parents=True, exist_ok=True) unpack_archive(cache.path, self.mathlib_folder, oleans_only=False) if cache.rev != repo.head.commit: # If the commit we unpacked isn't HEAD, then there might be some # zombie olean files around. It is probably safe, but slower, to do # this unconditionally. self.delete_zombies() # Let's now touch oleans, just in case touch_oleans(self.mathlib_folder) def mk_cache(self, force: bool = False) -> None: """Cache oleans for this project.""" if not self.rev: raise ValueError('This project has no git commit.') if not self.repo: raise LeanProjectError('This project has no git repository.') rev = self.rev if self.is_dirty: raise LeanProjectError('Unable to make a cache for a dirty ' 'repository. Commit or stash first.') tgt_folder = DOT_MATHLIB if self.is_mathlib else self.directory/'_cache' tgt_folder.mkdir(exist_ok=True) archive = tgt_folder/(str(self.rev) + '.tar.xz') if archive.exists() and not force: log.info('Cache for revision {} already exists, use --force to replace it.'.format(self.rev)) return pack(self.directory, filter(Path.exists, [self.src_directory, self.directory/'test']), archive) def get_cache(self, rev: Optional[str] = None, fallback: CacheFallback = CacheFallback.SHOW) -> None: """Tries to get olean cache for the current project. Will raise LeanDownloadError or FileNotFoundError if no archive exists. """ if not self.repo: raise LeanProjectError('This project has no git repository.') if self.is_mathlib: cache_locator = CacheLocator(self.name, self.repo, self.cache_url, DOT_MATHLIB, force_download=self.force_download) else: # TODO: support remote caches for non-mathlib projects cache_locator = CacheLocator(self.name, self.repo, None, self.directory/'_cache', force_download=self.force_download) commit = self.repo.rev_parse(rev) if rev is not None else self.repo.head.commit cache = cache_locator.find_local_with_fallback(commit, fallback) log.info("Applying cache") unpack_archive(cache.path, self.directory, oleans_only=True) if cache.rev != self.repo.head.commit: # If the commit we unpacked isn't HEAD, then there might be some # zombie olean files around. It is probably safe, but slower, to do # this unconditionally. self.delete_zombies() # Let's now touch oleans, just in case touch_oleans(self.directory) @classmethod def from_git_url(cls, url: str, target: str = '', branch: str = '', create_branch: bool = False, cache_url: str = '', force_download: bool = False) -> 'LeanProject': """Download a Lean project using git and prepare mathlib if needed.""" log.info('Cloning from ' + url) target = target or url.split('/')[-1].split('.')[0] repo = Repo.clone_from(url, target) if create_branch and branch: try: repo.git.checkout('HEAD', '-b', branch) except (IndexError, GitCommandError): log.error('Cannot create new git branch') shutil.rmtree(target) raise elif branch: try: repo.remote('origin').fetch(branch) repo.git.checkout(branch) except (IndexError, GitCommandError) as err: log.error('Invalid git branch') shutil.rmtree(target) raise err proj = cls.from_path(Path(repo.working_dir), cache_url, force_download) proj.run_echo(['leanpkg', 'configure']) if 'mathlib' in proj.deps or proj.is_mathlib: proj.get_mathlib_olean() return proj @classmethod def new(cls, path: Path = Path('.'), cache_url: str = '', force_download: bool = False) -> 'LeanProject': """Create a new Lean project and prepare mathlib.""" if path == Path('.'): subprocess.run(['leanpkg', 'init', path.absolute().name], check=True) else: if path.exists(): raise FileExistsError('Directory ' + str(path) + ' already exists') subprocess.run(['leanpkg', 'new', str(path)], check=True) proj = cls.from_path(path, cache_url, force_download) proj.lean_version = mathlib_lean_version() proj.write_config() proj.add_mathlib() assert proj.repo proj.repo.git.checkout('-b', 'master') return proj def run(self, args: List[str]) -> str: """Run a command in the project directory, and returns stdout + stderr. args is a list as in subprocess.run""" return subprocess.run(args, cwd=str(self.directory), stderr=subprocess.STDOUT, stdout=subprocess.PIPE, check=True).stdout.decode() def run_echo(self, args: List[str]) -> None: """Run a command in the project directory, letting stdin and stdout flow. args is a list as in subprocess.run""" subprocess.run(args, cwd=str(self.directory), check=True) def clean(self) -> None: src_dir = self.directory/self.pkg_config['path'] test_dir = self.directory/'test' if src_dir.exists(): clean(src_dir) else: raise InvalidLeanProject( "Directory {} specified by 'path' does not exist".format(src_dir)) if test_dir.exists(): clean(test_dir) def delete_zombies(self) -> None: src_dir = self.directory/self.pkg_config['path'] test_dir = self.directory/'test' if src_dir.exists(): delete_zombies(src_dir) else: raise InvalidLeanProject( "Directory {} specified by 'path' does not exist".format(src_dir)) if test_dir.exists(): delete_zombies(test_dir) def build(self) -> None: log.info('Building project ' + self.name) if not self.is_mathlib: self.clean_mathlib_dep() self.run_echo(['leanpkg', 'build']) def clean_mathlib_dep(self) -> None: """Restore git sanity in a mathlib dependency""" assert not self.is_mathlib if self.mathlib_folder.exists(): mathlib = self.mathlib_repo mathlib.head.reset(working_tree=True) mathlib.git.clean('-fd') else: self.run_echo(['leanpkg', 'configure']) def upgrade_mathlib(self) -> None: """Upgrade mathlib in the project. In case this project is mathlib, we assume we are already on the branch we want. """ if self.is_mathlib: assert self.repo try: rem = next(remote for remote in self.repo.remotes if any('leanprover' in url for url in remote.urls)) log.info('Pulling...') rem.pull(self.repo.active_branch) except (StopIteration, GitCommandError): log.info("Couldn't pull from a relevant git remote. " "You may try to git pull manually and then " "run `leanproject get-cache`") return self.rev = self.repo.commit().hexsha else: self.clean_mathlib_dep() if self.upgrade_lean: mathlib_lean = mathlib_lean_version() if mathlib_lean > self.lean_version: self.lean_version = mathlib_lean self.write_config() self.run_echo(['leanpkg', 'upgrade']) self.read_config() self.get_mathlib_olean() def add_mathlib(self) -> None: """Add mathlib to the project.""" if 'mathlib' in self.deps: log.info('This project already depends on mathlib') return log.info('Adding mathlib') if self.upgrade_lean: self.lean_version = mathlib_lean_version() self.write_config() self.run_echo(['leanpkg', 'add', 'leanprover-community/mathlib']) self.read_config() self.get_mathlib_olean() def setup_git_hooks(self) -> None: if self.repo is None: raise LeanProjectError('This project has no git repository.') hook_dir = Path(self.repo.git_dir)/'hooks' src = Path(__file__).parent log.info('This script will copy post-commit and post-checkout scripts to ', hook_dir) rep = input("Do you want to proceed (y/n)? ") if rep in ['y', 'Y']: shutil.copy(str(src/'post-commit'), str(hook_dir)) mode = (hook_dir/'post-commit').stat().st_mode (hook_dir/'post-commit').chmod(mode | stat.S_IXUSR) shutil.copy(str(src/'post-checkout'), str(hook_dir)) mode = (hook_dir/'post-checkout').stat().st_mode (hook_dir/'post-checkout').chmod(mode | stat.S_IXUSR) log.info("Successfully copied scripts") else: log.info("Cancelled...") def check_timestamps(self) -> Tuple[bool, bool]: """Check that core and mathlib oleans are more recent than their sources. Return a tuple (core_ok, mathlib_ok)""" try: mathlib_ok = all(p.stat().st_mtime < p.with_suffix('.olean').stat().st_mtime for p in (self.mathlib_folder/'src').glob('**/*.lean')) except FileNotFoundError: mathlib_ok = False return (check_core_timestamps(self.toolchain), mathlib_ok) @property def import_graph(self) -> 'ImportGraph': # Importing networkx + numpy is slow, so don't do it until this function # is called. from mathlibtools.import_graph import ImportGraph if self._import_graph: return self._import_graph G = ImportGraph(self.directory) for path in self.src_directory.glob('**/*.lean'): rel = path.relative_to(self.src_directory) label = str(rel.with_suffix('')).replace(os.sep, '.') G.add_node(label) imports = self.run(['lean', '--deps', str(path)]) for imp in map(Path, imports.split()): try: imp_rel = imp.relative_to(self.src_directory.resolve()) except ValueError: # This import is not from the project continue imp_label = str(imp_rel.with_suffix('')).replace(os.sep, '.') G.add_edge(imp_label, label) for node in G: G.nodes[node]['label'] = node self._import_graph = G return G def reduce_imports(self, file: str) -> Iterable[Tuple[str, List[str]]]: """ An iterator over files with removable imports, for each file yielding a list of removable imports in the format `("source.file", ["removable.import", "another.removable.import"])`. """ # Importing networkx is slow, so don't do it until this function # is called. import networkx as nx # type: ignore G = self.import_graph if file: G = G.ancestors(file) H = nx.transitive_reduction(G) if file: fs = [file] else: fs = G.nodes for f in fs: if f == "all": continue Gf = [e for e in G.edges if e[1] == f] Hf = [e for e in H.edges if e[1] == f] o = [e[0] for e in Gf if e not in Hf] if o: yield (f, o) def reduce_imports_sed(self, file: str) -> Iterable[str]: for src, removable in self.reduce_imports(file): for r in removable: # probably not the right command on osx yield "sed -i '/^import {line}$/d' src/{file}.lean".format(file=src.replace(".","/"), line=r) def make_all(self) -> None: """Creates all.lean importing everything from the project""" with (self.src_directory/'all.lean').open('w') as all_file: for path in self.src_directory.glob('**/*.lean'): rel = path.relative_to(self.src_directory).with_suffix('') if rel == Path('all'): continue all_file.write('import ' + ".".join(map(escape_identifier, rel.parts)) + '\n') def list_decls(self) -> Dict[str, DeclInfo]: """Collect declarations seen from this project, as a dictionary of DeclInfo""" all_exists = (self.src_directory/'all.lean').exists() list_decls_lean = self.src_directory/'list_decls.lean' try: list_decls_lean.unlink() except FileNotFoundError: pass log.info('Gathering imports') self.make_all() imports = (self.src_directory/'all.lean').read_text() decls_lean = (Path(__file__).parent/'decls.lean').read_text() list_decls_lean.write_text(imports+decls_lean) log.info('Collecting declarations') self.run_echo(['lean', '--run', str(list_decls_lean)]) data = yaml.safe_load((self.directory/'decls.yaml').open()) list_decls_lean.unlink() if not all_exists: (self.src_directory/'all.lean').unlink() decls = dict() for name, val in data.items(): fname = val['File'] line = val['Line'] if fname is None or line is None: continue path = Path(fname).absolute().resolve() if '_target' in fname: path = path.relative_to(self.directory/'_target'/'deps') origin = path.parts[0] path = path.relative_to(Path(origin)/'src') elif '.elan' in fname: origin = 'core' parts = path.parts path = Path('/'.join(parts[parts.index('.elan')+7:])) else: origin = self.name path = path.relative_to(self.src_directory) decls[name] = DeclInfo(origin, path, int(val['Line'])) return decls def pickle_decls(self, target: Path) -> None: """Safe declaration into a pickle file target""" with target.open('wb') as f: pickle.dump(self.list_decls(), f, pickle.HIGHEST_PROTOCOL) def pr(self, branch_name: str, force: bool = False) -> None: """ Prepare to work on a mathlib pull-request on a new branch. This will check for a clean working copy unless force is True. """ if self.is_dirty and not force: raise LeanDirtyRepo if not self.is_mathlib: raise LeanProjectError('This operation is for mathlib only.') if not self.repo: raise LeanProjectError('This project has no git repository.') if branch_name in self.repo.branches: raise LeanProjectError(f'The branch {branch_name} already exists, ' 'please choose another name.') log.info('Checking out master...') self.repo.git.checkout('master') origin = self.repo.remote().name self.upgrade_mathlib() log.info('Checking out new branch...') self.repo.git.checkout('-b', branch_name) log.info(f'Setting remote tracking to {origin}...') rem_ref = RemoteReference(self.repo, f"refs/remotes/{origin}/{branch_name}") self.repo.head.reference.set_tracking_branch(rem_ref) log.info('Done.') def rebase(self, force: bool = False) -> None: """ On mathlib, update master, get oleans and rebase current branch. This will check for a clean working copy unless force is True. """ if self.is_dirty and not force: raise LeanDirtyRepo('Cannot rebase because repository is dirty') if not self.is_mathlib: raise LeanProjectError('This operation is for mathlib only.') if not self.repo: raise LeanProjectError('This project has no git repository.') branch = self.repo.active_branch if branch.name == 'master': raise LeanProjectError('This does not make sense now ' 'since you are on master.') log.info('Checking out master...') self.repo.git.checkout('master') self.upgrade_mathlib() log.info(f'Checking out {branch}...') self.repo.git.checkout(branch) log.info('Rebasing...') self.repo.git.rebase('master') log.info('Done') def pull(self, remote: str='origin') -> None: """ Pull from the given remote and get mathlib oleans. """ if self.is_dirty: raise LeanDirtyRepo('Cannot pull because repository is dirty') old_mathlib = self.mathlib_rev log.info(f"Pulling from {remote}") self.repo.remote(remote).pull(self.repo.active_branch) self.read_config() if self.is_mathlib: self.get_cache() else: if self.mathlib_rev != old_mathlib: log.info("Updating mathlib") self.run_echo(['leanpkg', 'configure']) self.get_mathlib_olean() mathlib-tools-1.1.0/mathlibtools/post-checkout000077500000000000000000000012021412135126000214740ustar00rootroot00000000000000#!/bin/sh # SourceTree on OS X provides a defective path that doesn't contain python # https://jira.atlassian.com/browse/SRCTREE-6540 export PATH=$PATH:/usr/local/bin OLD_HEAD=$1 NEW_HEAD=$2 CHANGED_BRANCH=$3 if [ "$CHANGED_BRANCH" -eq "1" ]; then if /usr/bin/env python3 -c ""; then echo "Trying to fetch cached olean" leanproject get-cache # If the current project is not called "mathlib", then get a mathlib cache. leanpkg dump | grep --quiet 'name = "mathlib"' if [ $? -ne 0 ]; then leanproject get-mathlib-cache fi leanproject delete-zombies else echo "'env python3' failed; not running post-checkout hook" fi fi mathlib-tools-1.1.0/mathlibtools/post-commit000077500000000000000000000007151412135126000211670ustar00rootroot00000000000000#!/bin/sh # SourceTree on OS X provides a defective path that doesn't contain python # https://jira.atlassian.com/browse/SRCTREE-6540 export PATH=$PATH:/usr/local/bin # hooks are sometimes run from git GUIs with a limited environment # if python3 isn't available, we don't generate an error; just a message if /usr/bin/env python3 -c ""; then echo "Caching olean..." leanproject mk-cache else echo "'env python3' failed; not running post-commit hook" fi mathlib-tools-1.1.0/pyinstaller-requirements.txt000066400000000000000000000006701412135126000221150ustar00rootroot00000000000000altgraph==0.17 atomicwrites==1.4.0 certifi==2020.12.5 chardet==4.0.0 click==7.1.2 decorator==4.4.2 Deprecated==1.2.11 future==0.18.2 gitdb==4.0.5 GitPython==3.1.14 idna==2.10 networkx==2.5 pefile==2019.4.18 pydot==1.4.2 PyGithub==1.54.1 pyinstaller==4.3 pyinstaller-hooks-contrib==2021.1 PyJWT==1.7.1 pyparsing==2.4.7 pywin32-ctypes==0.2.0 PyYAML==5.4.1 requests==2.25.1 smmap==3.0.5 toml==0.10.2 tqdm==4.59.0 urllib3==1.26.5 wrapt==1.12.1 mathlib-tools-1.1.0/scripts/000077500000000000000000000000001412135126000157515ustar00rootroot00000000000000mathlib-tools-1.1.0/scripts/deploy_nightly.sh000077500000000000000000000050751412135126000213510ustar00rootroot00000000000000set -e # fail on error # Only run on builds for pushes to the master branch. if ! [ "$TRAVIS_EVENT_TYPE" = "push" -a "$TRAVIS_BRANCH" = "master" ]; then exit 0 fi # Make sure we have access to secure Travis environment variables. if ! [ "$TRAVIS_SECURE_ENV_VARS" = "true" ]; then echo 'deploy_nightly.sh: Build is a push to master, but no secure env vars.' >&2 exit 1 # Something's wrong. fi git remote add mathlib "https://$GITHUB_TOKEN@github.com/leanprover-community/mathlib.git" git remote add nightly "https://$GITHUB_TOKEN@github.com/leanprover-community/mathlib-nightly.git" # After this point, we don't use any secrets in commands. set -x # echo commands git fetch nightly --tags # Create a tag name based on the current date. MATHLIB_VERSION_STRING="nightly-$(date -u +%F)" # Check if the tag already exists. If so, exit (successfully). # This way we create a release/update the lean-x.y.z branch # only once per day. if git rev-parse --verify -q $MATHLIB_VERSION_STRING; then exit 0 fi if command -v greadlink >/dev/null 2>&1; then # macOS readlink doesn't support -f option READLINK=greadlink else READLINK=readlink fi # Try to update the lean-x.y.z branch on mathlib. This could fail if # a subsequent commit has already pushed an update. git push mathlib HEAD:refs/heads/$LEAN_VERSION || \ echo "mathlib rejected push to branch $LEAN_VERSION; maybe it already has a later version?" >&2 # Push the commits to a branch on nightly and push a tag. git push nightly HEAD:"mathlib-$TRAVIS_BRANCH" || true git tag $MATHLIB_VERSION_STRING git push nightly tag $MATHLIB_VERSION_STRING # Travis can't publish releases to other repos (or stop mucking with the markdown description), so push releases directly export GOPATH=$($READLINK -f go) PATH=$PATH:$GOPATH/bin go get github.com/itchio/gothub # Build olean and script tarballs. OLEAN_ARCHIVE=mathlib-olean-$MATHLIB_VERSION_STRING.tar.gz SCRIPT_ARCHIVE=mathlib-scripts-$MATHLIB_VERSION_STRING.tar.gz tar czf $OLEAN_ARCHIVE src rm -rf mathlib-scripts cp -a scripts mathlib-scripts tar czf $SCRIPT_ARCHIVE mathlib-scripts ls *.tar.gz # Create a release associated with the tag and upload the tarballs. gothub release -u leanprover-community -r mathlib-nightly -t $MATHLIB_VERSION_STRING -d "Mathlib's .olean files and scripts" --pre-release gothub upload -u leanprover-community -r mathlib-nightly -t $MATHLIB_VERSION_STRING -n "$(basename $OLEAN_ARCHIVE)" -f "$OLEAN_ARCHIVE" gothub upload -u leanprover-community -r mathlib-nightly -t $MATHLIB_VERSION_STRING -n "$(basename $SCRIPT_ARCHIVE)" -f "$SCRIPT_ARCHIVE" mathlib-tools-1.1.0/scripts/detect_errors.py000066400000000000000000000003231412135126000211650ustar00rootroot00000000000000import itertools import sys for line in sys.stdin: sys.stdout.write(line) if 'error' in line: for line in itertools.islice(sys.stdin, 20): sys.stdout.write(line) sys.exit(1) mathlib-tools-1.1.0/scripts/install_debian.sh000077500000000000000000000010671412135126000212640ustar00rootroot00000000000000#! /bin/bash sudo apt install -y git curl python3 python3-pip python3-venv # The following test is needed in case VScode was installed by other # means (e.g. using Ubuntu snap) if ! which code; then wget -O code.deb https://go.microsoft.com/fwlink/?LinkID=760868 sudo apt install -y ./code.deb rm code.deb fi code --install-extension jroesch.lean wget https://raw.githubusercontent.com/leanprover/elan/master/elan-init.sh bash elan-init.sh -y rm elan-init.sh python3 -m pip install --user pipx python3 -m pipx ensurepath . ~/.profile pipx install mathlibtools mathlib-tools-1.1.0/scripts/install_macos.sh000077500000000000000000000010111412135126000211310ustar00rootroot00000000000000#!/bin/bash # Install Homebrew set -e if ! which brew > /dev/null; then # Install Homebrew /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)" else # Update it, in case it has been ages since it's been updated brew update fi brew install elan mathlibtools elan toolchain install stable elan default stable # Install and configure VS Code if ! which code > /dev/null; then brew install --cask visual-studio-code fi code --install-extension jroesch.lean mathlib-tools-1.1.0/scripts/lean_version.lean000066400000000000000000000004111412135126000212720ustar00rootroot00000000000000-- Print the current version of lean, using logic copied from leanpkg. import system.io def lean_version_string_core := let (major, minor, patch) := lean.version in sformat!("{major}.{minor}.{patch}") def main : io unit := io.put_str_ln lean_version_string_core mathlib-tools-1.1.0/scripts/leanpkg-example.toml000066400000000000000000000003211412135126000217140ustar00rootroot00000000000000[package] name = "qpf" version = "0.1" lean_version = "3.4.2" path = "src" [dependencies] mathlib = {git = "https://github.com/leanprover-community/mathlib", rev = "8e4542dafe1f170b9b48d2293ad71e80a18dac87"} mathlib-tools-1.1.0/scripts/zulip.py000077500000000000000000000010271412135126000174710ustar00rootroot00000000000000import zulip # Pass the path to your zuliprc file here. client = zulip.Client(config_file="~/.zuliprc") # Send a stream message request = { "type": "stream", "to": "Denmark", "subject": "Castle", "content": "I come not, friends, to steal away your hearts." } result = client.send_message(request) print(result) # Send a private message request = { "type": "private", "to": "scott@tqft.net", "content": "With mirth and laughter let old wrinkles come." } result = client.send_message(request) print(result)mathlib-tools-1.1.0/setup.py000066400000000000000000000024441412135126000160000ustar00rootroot00000000000000import setuptools from os import path this_directory = path.abspath(path.dirname(__file__)) with open(path.join(this_directory, "README.md"), encoding='utf-8') as fh: long_description = fh.read() with open(path.join(this_directory, 'mathlibtools', '_version.py'), encoding='utf-8') as f: exec(f.read()) setuptools.setup( name="mathlibtools", version=__version__, # from _version.py author="The mathlib community", description="Lean prover mathlib supporting tools.", long_description=long_description, long_description_content_type="text/markdown", url="https://github.com/leanprover-community/mathlib-tools", packages=setuptools.find_packages(), entry_points={ "console_scripts": [ "leanproject = mathlibtools.leanproject:safe_cli", ]}, package_data = { 'mathlibtools': ['post-commit', 'post-checkout', 'decls.lean'] }, classifiers=[ "Programming Language :: Python :: 3", "License :: OSI Approved :: Apache Software License", "Operating System :: OS Independent" ], python_requires='>=3.6', install_requires=['toml>=0.10.0', 'PyGithub', 'certifi', 'gitpython>=2.1.11', 'requests', 'Click', 'tqdm', 'networkx', 'pydot', 'PyYAML>=3.13', 'atomicwrites'] ) mathlib-tools-1.1.0/tests/000077500000000000000000000000001412135126000154245ustar00rootroot00000000000000mathlib-tools-1.1.0/tests/test_functional.py000066400000000000000000000052641412135126000212060ustar00rootroot00000000000000import os import re import subprocess from pathlib import Path def chdir(path): # Fighting old pythons... os.chdir(str(path)) LEAN_VERSION_RE = re.compile(r'.*lean_version = "(3.[5-9][^"]*).*"', re.DOTALL) MATHLIB_REV_RE = re.compile(r".*mathlib.* = 'rev': '([^']*)'.*", re.DOTALL) def fix_leanpkg_bug(): """Fix the leanpkg toolchain bug in current folder.""" leanpkg = Path('leanpkg.toml') conf = leanpkg.read_text() m = LEAN_VERSION_RE.match(conf) if m: ver = m.group(1) leanpkg.write_text(conf.replace(ver, 'leanprover-community/lean:'+ver)) # The next helper is currently unused, but could be used in later tests. def change_mathlib_rev(rev): """Change the mathlib SHA in current folder.""" leanpkg = Path('leanpkg.toml') conf = leanpkg.read_text() m = MATHLIB_REV_RE.match(conf) if m: old_rev = m.group(1) leanpkg.write_text(conf.replace(old_rev, rev)) def test_new(tmpdir): """Create a new package and check mathlib oleans are there.""" chdir(tmpdir) subprocess.run(['leanproject', 'new']) assert (tmpdir/'leanpkg.path').exists() assert (tmpdir/'_target'/'deps'/'mathlib'/'src'/'algebra'/'default.olean').exists() def test_add(tmpdir): """Add mathlib to a project and check mathlib oleans are there.""" chdir(tmpdir) subprocess.run(['leanpkg', 'init', 'project']) fix_leanpkg_bug() subprocess.run(['leanproject', 'add-mathlib']) assert (tmpdir/'_target'/'deps'/'mathlib'/'src'/'algebra'/'default.olean').exists() def test_upgrade_project(tmpdir): chdir(tmpdir) subprocess.run(['leanpkg', 'init', 'project']) fix_leanpkg_bug() leanpkg = Path('leanpkg.toml') leanpkg.write_text(leanpkg.read_text() + 'mathlib = {git = "https://github.com/leanprover-community/mathlib",' 'rev = "a9ed54ca0329771deab21d7574d7d19b417bf4a3"}') subprocess.run(['leanproject', 'upgrade-mathlib']) assert (tmpdir/'_target'/'deps'/'mathlib'/'src'/'algebra'/'default.olean').exists() def test_upgrade_mathlib(tmpdir): chdir(tmpdir) subprocess.run(['git', 'clone', 'https://github.com/leanprover-community/mathlib']) chdir(tmpdir/'mathlib') subprocess.run(['git', 'checkout', 'lean-3.5.1']) subprocess.run(['git', 'reset', '--hard', 'a9ed54ca0329771deab21d7574d7d19b417bf4a3']) subprocess.run(['leanproject', 'upgrade-mathlib']) assert (tmpdir/'mathlib'/'src'/'algebra'/'default.olean').exists() def test_get_tutorials(tmpdir): chdir(tmpdir) subprocess.run(['leanproject', 'get', 'tutorials']) assert (tmpdir/'tutorials'/'src').exists() assert (tmpdir/'tutorials'/'_target'/'deps'/'mathlib'/'src'/'algebra'/'default.olean').exists() mathlib-tools-1.1.0/tests/test_git_helpers.py000066400000000000000000000036641412135126000213530ustar00rootroot00000000000000import os import pytest from types import SimpleNamespace import git from mathlibtools.git_helpers import visit_ancestors @pytest.fixture def dummy_repo(tmp_path): r""" A -- B -- E -- I -- J -- L \ / / C --- F -- H \ / D ---- G --- K """ d = tmp_path / "repo" d.mkdir() repo = git.Repo.init(d) with repo.config_writer() as cw: cw.set_value("user", "name", "pytest") cw.set_value("user", "email", "<>") # workaround for https://github.com/gitpython-developers/GitPython/pull/1314 os.environ['USER'] = 'gitpython needs this to be here so it can ignore it' A = repo.index.commit("A") B = repo.index.commit("B", parent_commits=(A,)) C = repo.index.commit("C", parent_commits=(B,)) D = repo.index.commit("D", parent_commits=(C,)) E = repo.index.commit("E", parent_commits=(B,)) F = repo.index.commit("F", parent_commits=(C,)) G = repo.index.commit("G", parent_commits=(D,)) I = repo.index.commit("I", parent_commits=(E, F)) H = repo.index.commit("H", parent_commits=(F, G)) J = repo.index.commit("J", parent_commits=(I, H)) K = repo.index.commit("K", parent_commits=(G,)) L = repo.index.commit("L", parent_commits=(J,)) return repo @pytest.mark.parametrize(['match', 'exp_found', 'exp_visited'], [ ('L', 'L', ''), # finding the root prunes everything else ('BFG', 'GF', 'LJHIE'), # B is pruned ('K', '', 'LJHGDIFCEBA'), # no match, all iterated ]) def test_visit_ancestors(dummy_repo, match, exp_found, exp_visited): assert dummy_repo.head.commit.message == 'L' found = [] visited = [] for c, prune in visit_ancestors(dummy_repo.head.commit): if c.message in list(match): prune() found.append(c.message) else: visited.append(c.message) assert visited == list(exp_visited) assert found == list(exp_found) mathlib-tools-1.1.0/tests/test_parse_project_name.py000066400000000000000000000037471412135126000227100ustar00rootroot00000000000000from mathlibtools.leanproject import parse_project_name as P def test_name(): name, url, branch, is_url = P('tutorials') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == '' assert not is_url def test_org_name(): name, url, branch, is_url = P('leanprover-community/tutorials') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == '' assert not is_url def test_https(): name, url, branch, is_url = P('https://github.com/leanprover-community/tutorials.git') assert name == 'tutorials' assert url == 'https://github.com/leanprover-community/tutorials.git' assert branch == '' assert is_url def test_ssh(): name, url, branch, is_url = P('git@github.com:leanprover-community/tutorials.git') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == '' assert is_url def test_name_branch(): name, url, branch, is_url = P('tutorials:foo') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == 'foo' assert not is_url def test_org_name_branch(): name, url, branch, is_url = P('leanprover-community/tutorials:foo') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == 'foo' assert not is_url def test_https_branch(): name, url, branch, is_url = P('https://github.com/leanprover-community/tutorials.git:foo') assert name == 'tutorials' assert url == 'https://github.com/leanprover-community/tutorials.git' assert branch == 'foo' assert is_url def test_ssh_branch(): name, url, branch, is_url = P('git@github.com:leanprover-community/tutorials.git:foo') assert name == 'tutorials' assert url == 'git@github.com:leanprover-community/tutorials.git' assert branch == 'foo' assert is_url mathlib-tools-1.1.0/tests/test_python.py000066400000000000000000000002411412135126000203530ustar00rootroot00000000000000""" Tests API provided by `mathlibtools` to other Python scripts """ import mathlibtools def test_version(): assert isinstance(mathlibtools.__version__, str) mathlib-tools-1.1.0/tox.ini000066400000000000000000000003501412135126000155730ustar00rootroot00000000000000[tox] envlist = py36, py37, py38, py39, mypy [testenv] commands = pytest deps = pytest [testenv:mypy] basepython = python3.8 deps = mypy setenv = MYPYPATH={toxinidir} commands = mypy --install-types --non-interactive mathlibtools