pax_global_header00006660000000000000000000000064143657013060014517gustar00rootroot0000000000000052 comment=b2733981a81855c07a5ce4c7c0622f8ee7039004 pyct-0.5.0/000077500000000000000000000000001436570130600125005ustar00rootroot00000000000000pyct-0.5.0/.gitattributes000066400000000000000000000000311436570130600153650ustar00rootroot00000000000000__init__.py export-subst pyct-0.5.0/.github/000077500000000000000000000000001436570130600140405ustar00rootroot00000000000000pyct-0.5.0/.github/workflows/000077500000000000000000000000001436570130600160755ustar00rootroot00000000000000pyct-0.5.0/.github/workflows/build.yaml000066400000000000000000000067331436570130600200710ustar00rootroot00000000000000name: packages on: push: tags: - 'v[0-9]+.[0-9]+.[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+a[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+b[0-9]+' - 'v[0-9]+.[0-9]+.[0-9]+rc[0-9]+' # Dry-run only workflow_dispatch: schedule: - cron: '0 19 * * SUN' jobs: conda_build: name: Build Conda Packages runs-on: 'ubuntu-latest' defaults: run: shell: bash -l {0} env: CHANS: "-c pyviz" PKG_TEST_PYTHON: "--test-python=py37" PYCTDEV_ECOSYSTEM: conda steps: - uses: actions/checkout@v3 with: fetch-depth: "100" - name: Fetch run: git fetch --prune --tags -f - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" - name: Set output id: vars run: echo "tag=${GITHUB_REF#refs/*/}" >> $GITHUB_OUTPUT - name: conda setup run: | conda install -c pyviz "pyctdev>=0.5" doit ecosystem_setup - name: conda build run: | doit package_build --recipe=core $PKG_TEST_PYTHON --test-group=build_examples $CHANS doit package_build $PKG_TEST_PYTHON --test-group=cmd_examples $CHANS - name: dev deploy if: (github.event_name == 'push' && (contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) run: | doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --recipe=core doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev - name: main deploy if: (github.event_name == 'push' && !(contains(steps.vars.outputs.tag, 'a') || contains(steps.vars.outputs.tag, 'b') || contains(steps.vars.outputs.tag, 'rc'))) run: | doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main --recipe=core doit package_upload --token=${{ secrets.CONDA_UPLOAD_TOKEN }} --label=dev --label=main pip_build: name: Build PyPI Packages runs-on: 'ubuntu-latest' defaults: run: shell: bash -l {0} env: CHANS: "-c pyviz" PKG_TEST_PYTHON: "--test-python=py37" PYTHON_VERSION: "3.7" PYPI: "https://upload.pypi.org/legacy/" PYCTDEV_ECOSYSTEM: conda steps: - uses: actions/checkout@v3 with: fetch-depth: "100" - uses: conda-incubator/setup-miniconda@v2 with: miniconda-version: "latest" - name: Fetch run: git fetch --prune --tags -f - name: conda setup run: | conda install -c pyviz "pyctdev>=0.5" doit ecosystem_setup doit env_create $CHANS --python=$PYTHON_VERSION - name: env setup run: | conda activate test-environment doit develop_install $CHANS -o build doit pip_on_conda - name: pip build run: | conda activate test-environment doit ecosystem=pip package_build --test-group=build_examples $PKG_TEST_PYTHON --sdist-install-build-deps --sdist-run-tests doit ecosystem=pip package_build --test-group=cmd_examples $PKG_TEST_PYTHON --sdist-install-build-deps --sdist-run-tests - name: git status run: | git status git diff - name: pip upload if: github.event_name == 'push' run: | conda activate test-environment doit ecosystem=pip package_upload -u ${{ secrets.PPU }} -p ${{ secrets.PPP }} -r $PYPI pyct-0.5.0/.github/workflows/test.yaml000066400000000000000000000023111436570130600177350ustar00rootroot00000000000000name: tests on: push: branches: - main pull_request: branches: - '*' workflow_dispatch: schedule: - cron: '0 19 * * SUN' jobs: test_suite: name: Unit tests on ${{ matrix.os }} with Python ${{ matrix.python-version }} runs-on: ${{ matrix.os }} strategy: fail-fast: false matrix: os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] timeout-minutes: 45 defaults: run: shell: bash -l {0} # env: # SETUPTOOLS_ENABLE_FEATURES: "legacy-editable" steps: - uses: actions/checkout@v3 with: fetch-depth: "100" - name: fetch tags run: git fetch --tags - uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: setup run: | python3 -m pip install --upgrade setuptools pip wheel python3 -m pip install "tox<4" tox-gh-actions - name: flakes run: tox -e flakes - name: unit run: tox -e unit - name: _cmd_examples run: tox -e cmd_examples - name: _build_examples run: tox -e build_examples pyct-0.5.0/.gitignore000066400000000000000000000002101436570130600144610ustar00rootroot00000000000000*~ *.pyc /dist *.egg-info pip-wheel-metadata .pytest_cache *__pycache__ .venv venv/ # pyct .doit.db .tox # autover */.version build/ pyct-0.5.0/LICENSE.txt000066400000000000000000000027071436570130600143310ustar00rootroot00000000000000Copyright (c) 2017-18, PyViz All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. * Neither the name of cube-explorer nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. pyct-0.5.0/MANIFEST.in000066400000000000000000000000741436570130600142370ustar00rootroot00000000000000include pyct/.version include LICENSE.txt include README.md pyct-0.5.0/README.md000066400000000000000000000134471436570130600137700ustar00rootroot00000000000000# pyct A utility package that includes: 1. **pyct.cmd**: Makes various commands available to other packages. (Currently no sophisticated plugin system, just a try import/except in the other packages.) The same commands are available from within python. Can either add new subcommands to an existing argparse based command if the module has an existing command, or create the entire command if the module has no existing command. Currently, there are commands for copying examples and fetching data. See 2. **pyct.build**: Provides various commands to help package building, primarily as a convenience for project maintainers. ## pyct.cmd To install pyct with the dependencies required for pyct.cmd: `pip install pyct[cmd]` or `conda install -c pyviz pyct`. An example of how to use in a project: https://github.com/holoviz/geoviews/blob/main/geoviews/__main__.py Once added, users can copy the examples of a package and download the required data with the `examples` command: ``` $ datashader examples --help usage: datashader examples [-h] [--path PATH] [-v] [--force] [--use-test-data] optional arguments: -h, --help show this help message and exit --path PATH location to place examples and data -v, --verbose --force if PATH already exists, force overwrite existing examples if older than source examples. ALSO force any existing data files to be replaced --use-test-data Use data's test files, if any, instead of fetching full data. If test file not in '.data_stubs', fall back to fetching full data. ``` To copy the examples of e.g. datashader but not download the data, there's a `copy-examples` command: ``` usage: datashader copy-examples [-h] [--path PATH] [-v] [--force] optional arguments: -h, --help show this help message and exit --path PATH where to copy examples -v, --verbose --force if PATH already exists, force overwrite existing files if older than source files ``` And to download the data only, the `fetch-data` command: ``` usage: datashader fetch-data [-h] [--path PATH] [--datasets DATASETS] [-v] [--force] [--use-test-data] optional arguments: -h, --help show this help message and exit --path PATH where to put data --datasets DATASETS *name* of datasets file; must exist either in path specified by --path or in package/examples/ -v, --verbose --force Force any existing data files to be replaced --use-test-data Use data's test files, if any, instead of fetching full data. If test file not in '.data_stubs', fall back to fetching full data. ``` Can specify different 'datasets' file: ``` $ cat earthsim-examples/test.yml --- data: - url: http://s3.amazonaws.com/datashader-data/Chesapeake_and_Delaware_Bays.zip title: 'Depth data for the Chesapeake and Delaware Bay region of the USA' files: - Chesapeake_and_Delaware_Bays.3dm $ earthsim fetch-data --path earthsim-examples --datasets-filename test.yml Downloading data defined in /tmp/earthsim-examples/test.yml to /tmp/earthsim-examples/data Skipping Depth data for the Chesapeake and Delaware Bay region of the USA ``` Can use smaller files instead of large ones by using the `--use-test-data` flag and placing a small file with the same name in `examples/data/.data_stubs`: ``` $ tree examples/data -a examples/data ├── .data_stubs │   └── nyc_taxi_wide.parq └── diamonds.csv $ cat examples/dataset.yml data: - url: http://s3.amazonaws.com/datashader-data/nyc_taxi_wide.parq title: 'NYC Taxi Data' files: - nyc_taxi_wide.parq - url: http://s3.amazonaws.com/datashader-data/maccdc2012_graph.zip title: 'National CyberWatch Mid-Atlantic Collegiate Cyber Defense Competition' files: - maccdc2012_nodes.parq - maccdc2012_edges.parq - maccdc2012_full_nodes.parq - maccdc2012_full_edges.parq $ pyviz fetch-data --path=examples --use-test-data Fetching data defined in /tmp/pyviz/examples/datasets.yml and placing in /tmp/pyviz/examples/data Copying test data file '/tmp/pyviz/examples/data/.data_stubs/nyc_taxi_wide.parq' to '/tmp/pyviz/examples/data/nyc_taxi_wide.parq' No test file found for: /tmp/pyviz/examples/data/.data_stubs/maccdc2012_nodes.parq. Using regular file instead Downloading National CyberWatch Mid-Atlantic Collegiate Cyber Defense Competition 1 of 1 [################################] 59/59 - 00:00:00 ``` To clean up any potential test files masquerading as real data use `clean-data`: ``` usage: pyviz clean-data [-h] [--path PATH] optional arguments: -h, --help show this help message and exit --path PATH where to clean data ``` ## pyct.build Currently provides a way to package examples with a project, by copying an examples folder into the package directory whenever setup.py is run. The way this works is likely to change in the near future, but is provided here as the first step towards unifying/simplifying the maintenance of a number of pyviz projects. ## pyct report Provides a way to check the package versions in the current environment using: 1. A console script (entry point): `pyct report [packages]`, or 2. A python function: `import pyct; pyct.report(packages)` The python function can be particularly useful for e.g. jupyter notebook users, since it is the packages in the current kernel that we usually care about (not those in the environment from which jupyter notebook server/lab was launched). Note that `packages` above can include the name of any Python package (returning the `__version__`), along with the special cases `python` or `conda` (returning the version of the command-line tool) or `system` (returning the OS version). pyct-0.5.0/conda.recipe/000077500000000000000000000000001436570130600150325ustar00rootroot00000000000000pyct-0.5.0/conda.recipe/core/000077500000000000000000000000001436570130600157625ustar00rootroot00000000000000pyct-0.5.0/conda.recipe/core/meta.yaml000066400000000000000000000012311436570130600175710ustar00rootroot00000000000000{% set sdata = load_setup_py_data() %} package: name: pyct-core version: {{ sdata['version'] }} source: path: ../.. build: noarch: python script: python setup.py install --single-version-externally-managed --record=record.txt requirements: run_constrained: - pyct {{ sdata['version'] }} host: - python - setuptools >=30.3.0 - param >=1.7.0 run: - python {{ sdata['python_requires'] }} {% for dep in sdata.get('install_requires',{}) %} - {{ dep }} {% endfor %} test: imports: - pyct - pyct.build about: home: {{ sdata['url'] }} summary: {{ sdata['description'] }} license: {{ sdata['license'] }} pyct-0.5.0/conda.recipe/meta.yaml000066400000000000000000000017121436570130600166450ustar00rootroot00000000000000{% set sdata = load_setup_py_data() %} package: name: pyct version: {{ sdata['version'] }} source: path: .. build: noarch: python entry_points: {% for group,epoints in sdata.get("entry_points",{}).items() %} {% for entry_point in epoints %} - {{ entry_point }} {% endfor %} {% endfor %} requirements: host: - python - setuptools >=30.3.0 - param >=1.7.0 run: - python {{ sdata['python_requires'] }} - pyct-core ={{ sdata['version'] }} {% for dep in sdata['extras_require']['cmd'] %} - {{ dep }} {% endfor %} test: imports: - pyct - pyct.cmd requires: {% for dep in sdata['extras_require']['tests'] %} - "{{ dep }}" {% endfor %} commands: - pytest -v --pyargs pyct - pyct --help - pyct --version - pyct report --help - pyct report pyct python about: home: {{ sdata['url'] }} summary: {{ sdata['description'] }} license: {{ sdata['license'] }} pyct-0.5.0/dodo.py000066400000000000000000000006271436570130600140040ustar00rootroot00000000000000from pyctdev import * # noqa: api def task_pip_on_conda(): """Experimental: provide pip build env via conda""" return {'actions':[ # some ecosystem=pip build tools must be installed with conda when using conda... 'conda install -y pip twine wheel', # ..and some are only available via conda-forge 'conda install -y -c conda-forge tox "virtualenv<=20.4.7"', ]} pyct-0.5.0/pyct/000077500000000000000000000000001436570130600134575ustar00rootroot00000000000000pyct-0.5.0/pyct/__init__.py000066400000000000000000000004241436570130600155700ustar00rootroot00000000000000import param NAME = "pyct" from .report import report # noqa: api # version comes from git if available, otherwise from .version file __version__ = str(param.version.Version(fpath=__file__, archive_commit="b273398", reponame=NAME)) pyct-0.5.0/pyct/__main__.py000066400000000000000000000001441436570130600155500ustar00rootroot00000000000000def main(args=None): import pyct.cmd pyct.cmd.main() if __name__ == "__main__": main() pyct-0.5.0/pyct/build.py000066400000000000000000000034451436570130600151360ustar00rootroot00000000000000import os import shutil def examples(path, root, verbose=False, force=False): """ Copies the notebooks to the supplied path. """ filepath = os.path.abspath(os.path.dirname(root)) example_dir = os.path.join(filepath, './examples') if not os.path.exists(example_dir): example_dir = os.path.join(filepath, '../examples') if os.path.exists(path): if not force: print('%s directory already exists, either delete it or set the force flag' % path) return shutil.rmtree(path) ignore = shutil.ignore_patterns('.ipynb_checkpoints', '*.pyc', '*~') tree_root = os.path.abspath(example_dir) if os.path.isdir(tree_root): shutil.copytree(tree_root, path, ignore=ignore, symlinks=True) else: print('Cannot find %s' % tree_root) def get_setup_version(root, reponame): """ Helper to get the current version from either git describe or the .version file (if available) - allows for param to not be available. Normally used in setup.py as follows: >>> from pyct.build import get_setup_version >>> version = get_setup_version(__file__, reponame) # noqa """ import json filepath = os.path.abspath(os.path.dirname(root)) version_file_path = os.path.join(filepath, reponame, '.version') try: from param import version except: version = None if version is not None: return version.Version.setup_version(filepath, reponame, archive_commit="$Format:%h$") else: print("WARNING: param>=1.6.0 unavailable. If you are installing a package, this warning can safely be ignored. If you are creating a package or otherwise operating in a git repository, you should install param>=1.6.0.") return json.load(open(version_file_path, 'r'))['version_string'] pyct-0.5.0/pyct/cmd.py000066400000000000000000000437001436570130600146000ustar00rootroot00000000000000# TODO: various stuff just pasted together in one place; needs clean up # -*- coding: utf-8 -*- from __future__ import print_function, absolute_import, division from . import __version__ from .report import report import os import importlib import inspect import argparse import distutils.dir_util import shutil def _find_examples(name): module_path = os.path.dirname(inspect.getfile(importlib.import_module(name))) candidates = [ # installed package os.path.join(module_path,"examples"), # git repo os.path.join(module_path,"..","examples")] for candidate in candidates: if os.path.exists(candidate): return candidate raise ValueError("Could not find examples for %s at any of %s"%(name,candidates)) def examples(name,path,verbose=False,use_test_data=False,force=False): """ Copy examples and fetch data (if any) to the supplied path. See copy-examples and fetch-data for more flexibility. NOTE: force operates both on example and data over-writing pre-existing files. """ copy_examples(name, path, verbose, force) fetch_data(name,path,require_datasets=False,use_test_data=use_test_data,force=force) def copy_examples(name,path,verbose=False,force=False): """Copy examples to the supplied path.""" source = _find_examples(name) path = os.path.abspath(path) if os.path.exists(path) and not force: raise ValueError("Path %s already exists; please move it away, choose a different path, or use force."%path) if verbose: print("Copying examples from %s"%source) distutils.dir_util.copy_tree(source, path, verbose=verbose) print("Copied examples to %s"%path) """ Copyright (c) 2011, Kenneth Reitz Permission to use, copy, modify, and/or distribute this software for any purpose with or without fee is hereby granted, provided that the above copyright notice and this permission notice appear in all copies. THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. clint.textui.progress ~~~~~~~~~~~~~~~~~ This module provides the progressbar functionality. """ from collections import OrderedDict import glob import sys import tarfile import time import zipfile import yaml try: import requests except ImportError: requests = None # TODO # if requests is None: # print('this download script requires the requests module: conda install requests') # sys.exit(1) STREAM = sys.stderr BAR_TEMPLATE = '%s[%s%s] %i/%i - %s\r' MILL_TEMPLATE = '%s %s %i/%i\r' DOTS_CHAR = '.' BAR_FILLED_CHAR = '#' BAR_EMPTY_CHAR = ' ' MILL_CHARS = ['|', '/', '-', '\\'] # How long to wait before recalculating the ETA ETA_INTERVAL = 1 # How many intervals (excluding the current one) to calculate the simple moving # average ETA_SMA_WINDOW = 9 DATA_DIR = 'data' DATA_STUBS_DIR = '.data_stubs' class Bar(object): def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): self.done() return False # we're not suppressing exceptions def __init__(self, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR, filled_char=BAR_FILLED_CHAR, expected_size=None, every=1): '''Bar is a class for printing the status of downloads''' self.label = label self.width = width self.hide = hide # Only show bar in terminals by default (better for piping, logging etc.) if hide is None: try: self.hide = not STREAM.isatty() except AttributeError: # output does not support isatty() self.hide = True self.empty_char = empty_char self.filled_char = filled_char self.expected_size = expected_size self.every = every self.start = time.time() self.ittimes = [] self.eta = 0 self.etadelta = time.time() self.etadisp = self.format_time(self.eta) self.last_progress = 0 if (self.expected_size): self.show(0) def show(self, progress, count=None): if count is not None: self.expected_size = count if self.expected_size is None: raise Exception("expected_size not initialized") self.last_progress = progress if (time.time() - self.etadelta) > ETA_INTERVAL: self.etadelta = time.time() self.ittimes = \ self.ittimes[-ETA_SMA_WINDOW:] + \ [-(self.start - time.time()) / (progress+1)] self.eta = \ sum(self.ittimes) / float(len(self.ittimes)) * \ (self.expected_size - progress) self.etadisp = self.format_time(self.eta) x = int(self.width * progress / self.expected_size) if not self.hide: if ((progress % self.every) == 0 or # True every "every" updates (progress == self.expected_size)): # And when we're done STREAM.write(BAR_TEMPLATE % ( self.label, self.filled_char * x, self.empty_char * (self.width - x), progress, self.expected_size, self.etadisp)) STREAM.flush() def done(self): self.elapsed = time.time() - self.start elapsed_disp = self.format_time(self.elapsed) if not self.hide: # Print completed bar with elapsed time STREAM.write(BAR_TEMPLATE % ( self.label, self.filled_char * self.width, self.empty_char * 0, self.last_progress, self.expected_size, elapsed_disp)) STREAM.write('\n') STREAM.flush() def format_time(self, seconds): return time.strftime('%H:%M:%S', time.gmtime(seconds)) def bar(it, label='', width=32, hide=None, empty_char=BAR_EMPTY_CHAR, filled_char=BAR_FILLED_CHAR, expected_size=None, every=1): """Progress iterator. Wrap your iterables with it.""" count = len(it) if expected_size is None else expected_size with Bar(label=label, width=width, hide=hide, empty_char=BAR_EMPTY_CHAR, filled_char=BAR_FILLED_CHAR, expected_size=count, every=every) \ as bar: for i, item in enumerate(it): yield item bar.show(i + 1) def ordered_load(stream, Loader=yaml.Loader, object_pairs_hook=OrderedDict): class OrderedLoader(Loader): pass def construct_mapping(loader, node): loader.flatten_mapping(node) return object_pairs_hook(loader.construct_pairs(node)) OrderedLoader.add_constructor( yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, construct_mapping) return yaml.load(stream, OrderedLoader) class DirectoryContext(object): """ Context Manager for changing directories """ def __init__(self, path): self.old_dir = os.getcwd() self.new_dir = path def __enter__(self): os.chdir(self.new_dir) def __exit__(self, *args): os.chdir(self.old_dir) def _url_to_binary_write(url, output_path, title): '''Given a url, output_path and title, write the contents of a requests get operation to the url in binary mode and print the title of operation''' print('Downloading {0}'.format(title)) resp = requests.get(url, stream=True) try: with open(output_path, 'wb') as f: total_length = int(resp.headers.get('content-length')) for chunk in bar(resp.iter_content(chunk_size=1024), expected_size=(total_length/1024) + 1, every=1000): if chunk: f.write(chunk) f.flush() except: # Don't leave a half-written zip file if os.path.exists(output_path): os.remove(output_path) raise def _extract_downloaded_archive(output_path): '''Extract a local archive, e.g. zip or tar, then delete the archive''' if output_path.endswith("tar.gz"): with tarfile.open(output_path, "r:gz") as tar: tar.extractall() os.remove(output_path) elif output_path.endswith("tar"): with tarfile.open(output_path, "r:") as tar: tar.extractall() os.remove(output_path) elif output_path.endswith("tar.bz2"): with tarfile.open(output_path, "r:bz2") as tar: tar.extractall() os.remove(output_path) elif output_path.endswith("zip"): with zipfile.ZipFile(output_path, 'r') as zipf: zipf.extractall() os.remove(output_path) def _process_dataset(dataset, output_dir, here, use_test_data=False, force=False): '''Process each download spec in datasets.yml Typically each dataset list entry in the yml has "files" and "url" and "title" keys/values to show local files that must be present / extracted from a decompression of contents downloaded from the url. If a url endswith '/', then all files given are assumed to be added to the url pattern at the end ''' if not os.path.exists(output_dir): os.makedirs(output_dir) with DirectoryContext(output_dir): requires_download = False for f in dataset.get('files', []): if not os.path.exists(f): requires_download = True break if force is False and not requires_download: print('Skipping {0}'.format(dataset['title'])) return url = dataset['url'] title_fmt = dataset['title'] + ' {} of {}' if url.endswith('/'): urls = [url + f for f in dataset['files']] output_paths = [os.path.join(here, DATA_DIR, fname) for fname in dataset['files']] unpacked = ['.'.join(output_path.split('.')[:(-2 if output_path.endswith('gz') else -1)]) + '*' for output_path in output_paths] else: urls = [url] output_paths = [os.path.split(url)[1]] unpacked = dataset['files'] if not isinstance(unpacked, (tuple, list)): unpacked = [unpacked] zipped = zip(urls, output_paths, unpacked) for idx, (url, output_path, unpack) in enumerate(zipped): running_title = title_fmt.format(idx + 1, len(urls)) if force is False and (glob.glob(unpack) or os.path.exists(unpack.replace('*',''))): # Skip a file if a similar one is downloaded: # i.e. one that has same name but dif't extension print('Skipping {0}'.format(running_title)) continue test = os.path.join(output_dir, DATA_STUBS_DIR, unpack) if use_test_data and os.path.exists(test): target = os.path.join(output_dir, unpack) print("Copying test data file '{0}' to '{1}'".format(test, target)) shutil.copyfile(test, target) continue elif use_test_data and not os.path.exists(test): print("No test file found for: {}. Using regular file instead".format(test)) _url_to_binary_write(url, output_path, running_title) _extract_downloaded_archive(output_path) if requests is None: print('this download script requires the requests module: conda install requests') sys.exit(1) def fetch_data(name,path,datasets="datasets.yml",require_datasets=True,use_test_data=False,force=False): '''Fetch sample datasets as defined by path/datasets if it exists or else module's own examples/datasets otherwise. Datasets are placed in path/data ''' path = os.path.abspath(path) info_file = os.path.join(path,datasets) if not os.path.exists(info_file): info_file = os.path.join(_find_examples(name),datasets) if not os.path.exists(info_file) and require_datasets is False: print("No datasets to download") return print("Fetching data defined in %s and placing in %s"%(info_file,os.path.join(path,DATA_DIR))) # data is added later... with open(info_file) as f: info = ordered_load(f.read()) for topic, downloads in info.items(): output_dir = os.path.join(path, topic) for d in downloads: _process_dataset(d, output_dir, path, use_test_data=use_test_data, force=force) def clean_data(name, path): '''Remove up any data files that are copied from test files ''' path = os.path.abspath(path) if not os.path.exists(path): path = _find_examples(name) data_dir = os.path.join(path, DATA_DIR) test_dir = os.path.join(data_dir, DATA_STUBS_DIR) if not os.path.exists(test_dir) or len(os.listdir(test_dir)) == 0: print("No test files found") return for f in os.listdir(test_dir): data_file = os.path.join(data_dir, f) if not os.path.isfile(data_file): print("Test file was not copied to data:", f) continue test_file = os.path.join(test_dir, f) if os.path.isfile(test_file): data_s = os.path.getsize(data_file) test_s = os.path.getsize(test_file) if data_s == test_s: print("Removing copied test file:", f) os.remove(data_file) else: print("Size of test file {:.2e} did not match " "size of data file {:.2e}".format(test_s, data_s)) def _add_common_args(parser, name, *args): if '-v' in args: parser.add_argument('-v', '--verbose', action='count', default=0) if '--path' in args: parser.add_argument('--path',type=str, help='where to place output', default='{}-examples'.format(name)) def _set_defaults(parser, name, fn): parser.set_defaults(func=lambda args: fn(name, **{k: getattr(args,k) for k in vars(args) if k!='func'} )) # TODO: cmds=None defaults to 'all', basically, which is a bit confusing def add_commands(parser, name, cmds=None, args=None): """ Add all commands in pyct.cmd unless specific cmds are listed """ if cmds is None: cmds = ['examples','copy-examples','fetch-data','clean-data'] # use dict/reg instead if 'copy-examples' in cmds: eg_parser = parser.add_parser('copy-examples', help=inspect.getdoc(copy_examples)) eg_parser.add_argument('--force', action='store_true', help=('if PATH already exists, force overwrite existing ' 'files if older than source files')) _add_common_args(eg_parser, name, '-v', '--path') _set_defaults(eg_parser, name, copy_examples) if 'fetch-data' in cmds: d_parser = parser.add_parser('fetch-data', help=inspect.getdoc(fetch_data)) d_parser.add_argument('--datasets', type=str, default='datasets.yml', help=('*name* of datasets file; must exist either in path ' 'specified by --path or in package/examples/')) d_parser.add_argument('--force', action='store_true', help='Force any existing data files to be replaced') d_parser.add_argument('--use-test-data', action='store_true', help=("Use data's test files, if any, instead of fetching full data. " "If test file not in '.data_stubs', fall back to fetching full data.")) _add_common_args(d_parser, name, '--path') _set_defaults(d_parser, name, fetch_data) if 'examples' in cmds: egd_parser = parser.add_parser('examples', help=inspect.getdoc(examples)) egd_parser.add_argument('--force', action='store_true', help=('if PATH already exists, force overwrite existing examples if older ' 'than source examples. ALSO force any existing data files to be replaced')) egd_parser.add_argument('--use-test-data', action='store_true', help=("Use data's test files, if any, instead of fetching full data. " "If test file not in '.data_stubs', fall back to fetching full data.")) _add_common_args(egd_parser, name, '-v', '--path') _set_defaults(egd_parser, name, examples) if 'clean-data' in cmds: cd_parser = parser.add_parser('clean-data', help=inspect.getdoc(clean_data)) _add_common_args(cd_parser, name, '--path') _set_defaults(cd_parser, name, clean_data) def add_version(parser, name): mod = importlib.import_module(name) parser.add_argument('--version', action='version', version='%(prog)s ' + mod.__version__) def substitute_main(name, cmds=None, args=None): """ If module has no other commands, use this function to add all of the ones in pyct.cmd """ parser = argparse.ArgumentParser(description="%s commands"%name) subparsers = parser.add_subparsers(title='available commands') add_commands(subparsers, name, cmds, args) add_version(parser, name) args = parser.parse_args() args.func(args) if hasattr(args,'func') else parser.error("must supply command to run") def main(): parser = argparse.ArgumentParser(description="Commands relating to versioning") parser.add_argument('--version', action='version', version='%(prog)s '+__version__) subparsers = parser.add_subparsers(title='available commands') report_parser = subparsers.add_parser('report', help=inspect.getdoc(report)) report_parser.set_defaults(func=report) report_parser.add_argument('packages',metavar='package',type=str,nargs='+', help='name of package') args = parser.parse_args() if hasattr(args,'func'): args.func(*args.packages) else: parser.error("must supply command to run") pyct-0.5.0/pyct/report.py000077500000000000000000000047651436570130600153630ustar00rootroot00000000000000#!/usr/bin/env python # Import and print the library version and filesystem location for each Python package or shell command specified # # bash usage: $ report numpy pandas python conda # python usage: >>> from report import report; report("numpy","pandas","python","conda") from __future__ import print_function import os.path, importlib, subprocess, platform, sys def report(*packages): """Import and print location and version information for specified Python packages""" accepted_commands = ['python','conda'] for package in packages: loc = "not installed in this environment" ver = "unknown" try: module = importlib.import_module(package) loc = os.path.dirname(module.__file__) try: ver = str(module.__version__) except Exception: pass except (ImportError, ModuleNotFoundError): if package in accepted_commands: try: # See if there is a command by that name and check its --version if so try: loc = subprocess.check_output('command -v {}'.format(package), shell=True).decode().splitlines()[0].strip() except: # .exe in case powershell (otherwise wouldn't need it) loc = subprocess.check_output( 'where.exe {}'.format(package), shell=True).decode().splitlines()[0].strip() out = "" try: out = subprocess.check_output([package, '--version'], stderr=subprocess.STDOUT) except subprocess.CalledProcessError as e: out = e.output # Assume first word in output with a period and digits is the version for s in out.decode().split(): if '.' in s and str.isdigit(s[0]) and sum(str.isdigit(c) for c in s)>=2: ver=s.strip() break except: pass elif package == 'system': try: ver = platform.platform(terse=True) loc = "OS: " + platform.platform() except Exception: pass else: pass print("{0:30} # {1}".format(package + "=" + ver,loc)) def main(): report(*(sys.argv[1:])) if __name__ == "__main__": main() pyct-0.5.0/pyct/tests/000077500000000000000000000000001436570130600146215ustar00rootroot00000000000000pyct-0.5.0/pyct/tests/__init__.py000066400000000000000000000000001436570130600167200ustar00rootroot00000000000000pyct-0.5.0/pyct/tests/test_cmd.py000066400000000000000000000221531436570130600170000ustar00rootroot00000000000000import os import pytest import pyct.cmd from pyct.cmd import fetch_data, clean_data, copy_examples, examples # Same as in pyct/examples/datasets.yml DATASETS_CONTENT = u""" data: - url: this_should_never_be_used title: 'Test Data' files: - test_data.csv """ # Same as in pyct/examples/data/.data_stubs/test_data.csv TEST_FILE_CONTENT = u""" name,score,rank Alice,100.5,1 Bob,50.3,2 Charlie,25,3 """ REAL_FILE_CONTENT = u""" name,score,rank Alice,100.5,1 Bob,50.3,2 Charlie,25,3 Dave,28,4 Eve,25,3 Frank,75,9 """ EXAMPLE_CONTENT = u"""{ "cells": [ { "cell_type": "markdown", "metadata": {}, "source": [ "**NOTE:** This is a temporary notebook that gets created for tests." ] }, ], "metadata": { "language_info": { "name": "python", "pygments_lexer": "ipython3" } }, "nbformat": 4, "nbformat_minor": 2 } """ @pytest.fixture(autouse=True) def tmp_module(tmp_path): """This sets up a temporary directory structure meant to mimic a module """ project = tmp_path / "static_module" project.mkdir() examples = project / "examples" examples.mkdir() (examples / "Test_Example_Notebook.ipynb").write_text(EXAMPLE_CONTENT) (examples / "datasets.yml").write_text(DATASETS_CONTENT) (examples / "data").mkdir() (examples / "data" / ".data_stubs").mkdir() (examples / "data" / ".data_stubs" / "test_data.csv").write_text(TEST_FILE_CONTENT) return project @pytest.fixture(autouse=True) def monkeypatch_find_examples(monkeypatch, tmp_module): """Monkeypatching find examples to use a tmp examples. """ def _find_examples(name): return os.path.join(str(tmp_module), "examples") monkeypatch.setattr(pyct.cmd, '_find_examples', _find_examples) @pytest.fixture(scope='function') def tmp_project(tmp_path): project = tmp_path / "test_project" project.mkdir() return project @pytest.fixture(scope='function') def tmp_project_with_examples(tmp_path): project = tmp_path examples = project / "examples" examples.mkdir() datasets = examples / "datasets.yml" datasets.write_text(DATASETS_CONTENT) (examples / "data").mkdir() example = examples / "Test_Example_Notebook.ipynb" example.write_text(u"Fake notebook contents") return project @pytest.fixture(scope='function') def tmp_project_with_stubs(tmp_project_with_examples): project = tmp_project_with_examples data_stubs = project / "examples" / "data" / ".data_stubs" data_stubs.mkdir() return project @pytest.fixture(scope='function') def tmp_project_with_test_file(tmp_project_with_stubs): project = tmp_project_with_stubs data_stub = project / "examples" / "data" / ".data_stubs" / "test_data.csv" data_stub.write_text(TEST_FILE_CONTENT) return project def test_examples_with_use_test_data(tmp_project): project = tmp_project path = str(project / "examples") examples(name="pyct", path=path, use_test_data=True) assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() def test_examples_with_prexisting_content_in_target_raises_error(tmp_project_with_examples): project = tmp_project_with_examples path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) with pytest.raises(ValueError): examples(name="pyct", path=path, use_test_data=True) assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() assert (project / "examples" / "Test_Example_Notebook.ipynb").read_text() != EXAMPLE_CONTENT assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == REAL_FILE_CONTENT def test_examples_using_test_data_and_force_with_prexisting_content_in_target(tmp_project_with_examples): project = tmp_project_with_examples path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) examples(name="pyct", path=path, use_test_data=True, force=True) assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() assert (project / "examples" / "Test_Example_Notebook.ipynb").read_text() == EXAMPLE_CONTENT assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == TEST_FILE_CONTENT def test_copy_examples(tmp_project): project = tmp_project path = str(project / "examples") copy_examples(name="pyct", path=path) assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() def test_copy_examples_with_prexisting_content_in_target_raises_error(tmp_project_with_examples): project = tmp_project_with_examples path = str(project / "examples") with pytest.raises(ValueError): copy_examples(name="pyct", path=path) assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() assert (project / "examples" / "Test_Example_Notebook.ipynb").read_text() != EXAMPLE_CONTENT def test_copy_examples_using_force_with_prexisting_content_in_target(tmp_project_with_examples): project = tmp_project_with_examples path = str(project / "examples") copy_examples(name="pyct", path=path, force=True) assert (project / "examples" / "Test_Example_Notebook.ipynb").is_file() assert (project / "examples" / "Test_Example_Notebook.ipynb").read_text() == EXAMPLE_CONTENT def test_fetch_data_using_test_data_with_no_file_in_data_copies_from_stubs(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") fetch_data(name="pyct", path=path, use_test_data=True) assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == TEST_FILE_CONTENT def test_fetch_data_using_test_data_with_file_in_data_skips(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) fetch_data(name="pyct", path=path, use_test_data=True) assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == REAL_FILE_CONTENT def test_fetch_data_using_test_data_and_force_with_file_in_data_over_writes(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) fetch_data(name="pyct", path=path, use_test_data=True, force=True) assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == TEST_FILE_CONTENT def test_clean_data_when_data_file_is_real_does_nothing(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) clean_data(name="pyct", path=path) assert (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / "test_data.csv").read_text() == REAL_FILE_CONTENT def test_clean_data_when_data_file_is_from_stubs_removes_file_from_data(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(TEST_FILE_CONTENT) clean_data(name="pyct", path=path) assert not (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / ".data_stubs" / "test_data.csv").is_file() assert (project / "examples" / "data" / ".data_stubs" / "test_data.csv").read_text() == TEST_FILE_CONTENT def test_clean_data_when_file_not_in_data_does_nothing(tmp_project_with_test_file): project = tmp_project_with_test_file path = str(project / "examples") clean_data(name="pyct", path=path) assert not (project / "examples" / "data" / "test_data.csv").is_file() assert (project / "examples" / "data" / ".data_stubs" / "test_data.csv").is_file() assert (project / "examples" / "data" / ".data_stubs" / "test_data.csv").read_text() == TEST_FILE_CONTENT def test_clean_data_when_stubs_is_empty_does_nothing(tmp_project_with_stubs): project = tmp_project_with_stubs path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) clean_data(name="pyct", path=path) assert (project / "examples" / "data" / "test_data.csv").is_file() assert not (project / "examples" / "data" / ".data_stubs" / "test_data.csv").is_file() def test_clean_data_when_no_stubs_dir_does_nothing(tmp_project_with_examples): project = tmp_project_with_examples path = str(project / "examples") data = project / "examples" / "data" / "test_data.csv" data.write_text(REAL_FILE_CONTENT) clean_data(name="pyct", path=path) assert (project / "examples" / "data" / "test_data.csv").is_file() pyct-0.5.0/pyct/tests/test_report.py000066400000000000000000000034451436570130600175530ustar00rootroot00000000000000from unittest.mock import patch from pyct.report import report class TestModule: __version__ = "1.9.3" __file__ = "/mock/opt/anaconda3/envs/pyct/lib/python3.7/site-packages/param/__init__.py" @patch("importlib.import_module") @patch("builtins.print") def test_report_gives_package_version(mock_print, mock_import_module): module = TestModule() mock_import_module.return_value = module report("param") mock_print.assert_called_with('param=1.9.3 # /mock/opt/anaconda3/envs/pyct/lib/python3.7/site-packages/param') @patch("builtins.print") @patch("subprocess.check_output") def test_report_gives_conda_version(mock_check_output, mock_print): mock_check_output.side_effect = [b'/mock/opt/anaconda3/condabin/conda\n', b'conda 4.8.3\n'] report("conda") mock_print.assert_called_with("conda=4.8.3 # /mock/opt/anaconda3/condabin/conda") @patch("builtins.print") @patch("subprocess.check_output") def test_report_gives_python_version(mock_check_output, mock_print): mock_check_output.side_effect = [b'/mock/opt/anaconda3/envs/pyct/bin/python\n', b'Python 3.7.7\n'] report("python") mock_print.assert_called_with("python=3.7.7 # /mock/opt/anaconda3/envs/pyct/bin/python") @patch("builtins.print") @patch("platform.platform") def test_report_gives_system_version(mock_platform, mock_print): mock_platform.side_effect = ["Darwin-19.2.0", "Darwin-19.2.0-x86_64-i386-64bit"] report("system") mock_print.assert_called_with("system=Darwin-19.2.0 # OS: Darwin-19.2.0-x86_64-i386-64bit") @patch("builtins.print") def test_unknown_package_output(mock_print): report("fake_package") mock_print.assert_called_with("fake_package=unknown # not installed in this environment") pyct-0.5.0/pyproject.toml000066400000000000000000000001631436570130600154140ustar00rootroot00000000000000[build-system] requires = [ "param >=1.7.0", "setuptools >=61.0" ] build-backend = "setuptools.build_meta" pyct-0.5.0/setup.py000077500000000000000000000056471436570130600142310ustar00rootroot00000000000000from setuptools import setup, find_packages # unfortunately cannot avoid duplicating this from build.py def get_setup_version(root, reponame): """ Helper to get the current version from either git describe or the .version file (if available) - allows for param to not be available. Normally used in setup.py as follows: >>> from pyct.build import get_setup_version >>> version = get_setup_version(__file__, reponame) # noqa """ import os import json filepath = os.path.abspath(os.path.dirname(root)) version_file_path = os.path.join(filepath, reponame, '.version') try: from param import version except: version = None if version is not None: return version.Version.setup_version(filepath, reponame, archive_commit="$Format:%h$") else: print("WARNING: param>=1.6.0 unavailable. If you are installing a package, this warning can safely be ignored. If you are creating a package or otherwise operating in a git repository, you should install param>=1.6.0.") return json.load(open(version_file_path, 'r'))['version_string'] NAME = 'pyct' DESCRIPTION = 'Python package common tasks for users (e.g. copy examples, fetch data, ...)' setup_args = dict( name=NAME, version=get_setup_version(__file__, NAME), description=DESCRIPTION, long_description=open("README.md").read(), long_description_content_type="text/markdown", license='BSD 3-Clause License', license_files=['LICENSE.txt'], classifiers = [ 'License :: OSI Approved :: BSD License', 'Operating System :: OS Independent', 'Programming Language :: Python', 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Development Status :: 4 - Beta' ], author='HoloViz', author_email='developers@holoviz.org', maintainer='HoloViz', maintainer_email='developers@holoviz.org', url='https://github.com/pyviz-dev/{}'.format(NAME), project_urls = { 'Bug Tracker': 'https://github.com/pyviz-dev/{}/issues'.format(NAME), 'Source Code': 'https://github.com/pyviz-dev/{}'.format(NAME), }, include_package_data=True, packages=find_packages(), python_requires='>=3.7', install_requires=[ 'param >=1.7.0', ], extras_require={ 'cmd': [ 'pyyaml', 'requests' ], 'tests': [ 'flake8', 'pytest' ], 'doc': [ 'nbsite', 'sphinx_ioam_theme' ], 'build': [ "setuptools", "param >=1.7.0", ] }, entry_points = { 'console_scripts': [ 'pyct=pyct.__main__:main', ], } ) if __name__ == "__main__": setup(**setup_args) pyct-0.5.0/tox.ini000066400000000000000000000031671436570130600140220ustar00rootroot00000000000000[tox] envlist = {py37,py38,py39,py310,py311}-{flakes,unit,cmd_examples,build_examples,all}-{default}-{dev,pkg} build = wheel [gh-actions] python = 3.7: py37 3.8: py38 3.9: py39 3.10: py310 3.11: py311 [_flakes] commands = flake8 deps = .[tests] [_cmd_examples] commands = pytest pyct {envpython} -c "import pyct; pyct.report('pyct','python','system')" deps = .[tests,cmd] [_build_examples] # TODO: not much of a test yet... commands = {envpython} -c "from pyct.build import examples, get_setup_version" deps = .[tests] [_all] commands = {[_flakes]commands} {[_unit]commands} {[_cmd_examples]commands} {[_build_examples]commands} deps = .[examples,tests,cmd] [_unit] description = Run unit tests deps = .[tests,cmd] commands = pytest pyct pyct --help pyct --version pyct report --help pyct report pyct python {envpython} -m pyct --version [testenv] usedevelop = true changedir = {envtmpdir} commands = unit: {[_unit]commands} cmd_examples: {[_cmd_examples]commands} build_examples: {[_build_examples]commands} flakes: {[_flakes]commands} all: {[_all]commands} deps = unit: {[_unit]deps} cmd_examples: {[_cmd_examples]deps} build_examples: {[_build_examples]deps} flakes: {[_flakes]deps} all: {[_all]deps} [pytest] addopts = -v --pyargs norecursedirs = doc .git dist build _build .ipynb_checkpoints apps [flake8] ignore = E,W include = *.py exclude = .git,__pycache__,.tox,.eggs,*.egg,doc,dist,build,_build,.ipynb_checkpoints,run_test.py,.venv