pax_global_header00006660000000000000000000000064147434032010014510gustar00rootroot0000000000000052 comment=c312969913e1db16498725f84e3fe4ae71a35a21 beangulp-0.2.0/000077500000000000000000000000001474340320100133045ustar00rootroot00000000000000beangulp-0.2.0/.github/000077500000000000000000000000001474340320100146445ustar00rootroot00000000000000beangulp-0.2.0/.github/workflows/000077500000000000000000000000001474340320100167015ustar00rootroot00000000000000beangulp-0.2.0/.github/workflows/install.yaml000066400000000000000000000022461474340320100212370ustar00rootroot00000000000000name: install on: - push - pull_request jobs: install: runs-on: ubuntu-latest strategy: fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: # we support >= py37 # but beancount trunk only support py>=39 python-version: '3.9' # Install dependencies for the example importers - run: sudo apt install poppler-utils - run: python -m pip install . - name: Run smoke test run: | cd examples python import.py --help - name: Run example importers unit tests run: | cd examples python -m unittest - name: Run example importers tests run: | cd examples python importers/acme.py test tests/acme --verbose python importers/csvbank.py test tests/csvbank --verbose python importers/utrade.py test tests/utrade --verbose - name: Run example ledger.import run: | cd examples python import.py identify Downloads -v python import.py extract Downloads python import.py archive Downloads -n -o documents beangulp-0.2.0/.github/workflows/lint.yaml000066400000000000000000000005031474340320100205310ustar00rootroot00000000000000name: lint on: - push - pull_request jobs: lint: runs-on: ubuntu-latest strategy: fail-fast: false steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: '3.11' - run: pip install ruff - run: ruff check beangulp/ examples/ beangulp-0.2.0/.github/workflows/release.yaml000066400000000000000000000011571474340320100212110ustar00rootroot00000000000000name: release on: push: tags: - 'v[0-9]*' jobs: build: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - run: python -m pip install build - run: python -m build - uses: actions/upload-artifact@v4 with: path: dist/* upload: needs: build runs-on: ubuntu-latest environment: upload permissions: id-token: write steps: - uses: actions/download-artifact@v4 with: merge-multiple: true path: dist - uses: pypa/gh-action-pypi-publish@release/v1 with: attestations: false beangulp-0.2.0/.github/workflows/test.yaml000066400000000000000000000015251474340320100205470ustar00rootroot00000000000000name: test on: - push - pull_request jobs: test: runs-on: ubuntu-latest strategy: fail-fast: false matrix: python: - '3.8' - '3.9' - '3.10' - '3.11' - '3.12' - '3.13' beancount: - '~=2.3.5' - '~=3.0.0' - '@git+https://github.com/beancount/beancount.git' exclude: # Beancount dev version requires Python 3.9+ - beancount: '@git+https://github.com/beancount/beancount.git' python: '3.8' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - run: pip install 'beancount ${{ matrix.beancount }}' - run: pip install -r requirements.txt pytest - run: python -m pytest beangulp beangulp-0.2.0/.gitignore000066400000000000000000000000141474340320100152670ustar00rootroot00000000000000__pycache__ beangulp-0.2.0/LICENSE000066400000000000000000000432541474340320100143210ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. beangulp-0.2.0/README.rst000066400000000000000000000015421474340320100147750ustar00rootroot00000000000000beangulp: Importers Framework for Beancount ------------------------------------------- ``beangulp`` provides a framework for importing transactions into a Beancount ledger from account statements and other documents and for managing documents. ``beangulp`` is compatible with both Beancount 2.3 and the Beancount 3.0. ``beangulp`` is the evolution of ``beancount.ingest`` in Beancount 2.3 and replaces it in the Beancount 3.0 release. Documentation is available in form of code docstrings and `examples`__. The `documentation`__ for ``beancount.ingest`` provided a good introduction and is still partially relevant to ``beangulp``. The rest of the Beancount documentation can be found `here`__. __ https://github.com/beancount/beangulp/tree/master/examples/ __ https://beancount.github.io/docs/importing_external_data.html __ https://beancount.github.io/docs/ beangulp-0.2.0/beangulp/000077500000000000000000000000001474340320100151015ustar00rootroot00000000000000beangulp-0.2.0/beangulp/__init__.py000066400000000000000000000236471474340320100172260ustar00rootroot00000000000000"""Code to help identify, extract, and file external downloads. This package contains code to help you build importers and drive the process of identifying which importer to run on an externally downloaded file, extract transactions from them and file away these files under a clean and rigidly named hierarchy for preservation. """ __copyright__ = "Copyright (C) 2016,2018 Martin Blais" __license__ = "GNU GPLv2" import os import sys import warnings import click from beancount import loader from beangulp import archive from beangulp import cache # noqa: F401 from beangulp import exceptions from beangulp import extract from beangulp import identify from beangulp import utils from beangulp.importer import Importer, ImporterProtocol, Adapter def _walk(file_or_dirs, log): """Convenience wrapper around beangulp.utils.walk() Log the name of the file being processed and check the input file size against identify.FILE_TOO_LARGE_THRESHOLD for too large input. """ for filename in utils.walk(file_or_dirs): log(f'* {filename:}', nl=False) if os.path.getsize(filename) > identify.FILE_TOO_LARGE_THRESHOLD: log(' ... SKIP') continue yield filename @click.command('extract') @click.argument('src', nargs=-1, type=click.Path(exists=True, resolve_path=True)) @click.option('--output', '-o', type=click.File('w'), default='-', help='Output file.') @click.option('--existing', '-e', type=click.Path(exists=True), help='Existing Beancount ledger for de-duplication.') @click.option('--reverse', '-r', is_flag=True, help='Sort entries in reverse order.') @click.option('--failfast', '-x', is_flag=True, help='Stop processing at the first error.') @click.option('--quiet', '-q', count=True, help='Suppress all output.') @click.pass_obj def _extract(ctx, src, output, existing, reverse, failfast, quiet): """Extract transactions from documents. Walk the SRC list of files or directories and extract the ledger entries from each file identified by one of the configured importers. The entries are written to the specified output file or to the standard output in Beancount ledger format in sections associated to the source document. """ verbosity = -quiet log = utils.logger(verbosity, err=True) errors = exceptions.ExceptionsTrap(log) # Load the ledger, if one is specified. existing_entries = loader.load_file(existing)[0] if existing else [] extracted = [] for filename in _walk(src, log): with errors: importer = identify.identify(ctx.importers, filename) if not importer: log('') # Newline. continue # Signal processing of this document. log(' ...', nl=False) # Extract entries. entries = extract.extract_from_file(importer, filename, existing_entries) account = importer.account(filename) extracted.append((filename, entries, account, importer)) log(' OK', fg='green') if failfast and errors: break # Sort. extract.sort_extracted_entries(extracted) # Deduplicate. for filename, entries, account, importer in extracted: importer.deduplicate(entries, existing_entries) existing_entries.extend(entries) # Invoke hooks. for func in ctx.hooks: extracted = func(extracted, existing_entries) # Serialize entries. extract.print_extracted_entries(extracted, output) if errors: sys.exit(1) @click.command('archive') @click.argument('src', nargs=-1, type=click.Path(exists=True, resolve_path=True)) @click.option('--destination', '-o', metavar='DIR', type=click.Path(exists=True, file_okay=False, resolve_path=True), help='The destination documents tree root directory.') @click.option('--overwrite', '-f', is_flag=True, help='Overwrite destination files with the same name.') @click.option('--dry-run', '-n', is_flag=True, help='Just print where the files would be moved.') @click.option('--failfast', '-x', is_flag=True, help='Stop processing at the first error.') @click.option('--quiet', '-q', count=True, help='Suppress all output.') @click.pass_obj def _archive(ctx, src, destination, dry_run, overwrite, failfast, quiet): """Archive documents. Walk the SRC list of files or directories and move each file identified by one of the configured importers in a directory hierarchy mirroring the structure of the accounts associated to the documents and with a file name composed by the document date and document name returned by the importer. Documents are moved to their filing location only when no errors are encountered processing all the input files. Documents in the destination directory are not overwritten, unless the --force option is used. When the directory hierarchy root is not specified with the --destination DIR options, it is assumed to be directory in which the ingest script is located. """ # If the output directory is not specified, move the files at the # root where the import script is located. Providing this default # seems better than using a required option. if destination is None: import __main__ destination = os.path.dirname(os.path.abspath(__main__.__file__)) verbosity = -quiet log = utils.logger(verbosity, err=True) errors = exceptions.ExceptionsTrap(log) renames = [] for filename in _walk(src, log): with errors: importer = identify.identify(ctx.importers, filename) if not importer: log('') # Newline. continue # Signal processing of this document. log(' ...', nl=False) destpath = archive.filepath(importer, filename) # Prepend destination directory path. destpath = os.path.join(destination, destpath) # Check for destination filename collisions. collisions = [src for src, dst in renames if dst == destpath] if collisions: raise exceptions.Error('Collision in destination file path.', destpath) # Check if the destination file already exists. if not overwrite and os.path.exists(destpath): raise exceptions.Error('Destination file already exists.', destpath) renames.append((filename, destpath)) log(' OK', fg='green') log(f' {destpath:}') if failfast and errors: break # If there are any errors, stop here. if errors: log('# Errors detected: documents will not be filed.') sys.exit(1) if not dry_run: for filename, destpath in renames: archive.move(filename, destpath) @click.command('identify') @click.argument('src', nargs=-1, type=click.Path(exists=True, resolve_path=True)) @click.option('--failfast', '-x', is_flag=True, help='Stop processing at the first error.') @click.option('--verbose', '-v', is_flag=True, help='Show account information.') @click.pass_obj def _identify(ctx, src, failfast, verbose): """Identify files for import. Walk the SRC list of files or directories and report each file identified by one of the configured importers. When verbose output is requested, also print the account name associated to the document by the importer. """ log = utils.logger(verbose) errors = exceptions.ExceptionsTrap(log) for filename in _walk(src, log): with errors: importer = identify.identify(ctx.importers, filename) if not importer: log('') # Newline. continue # Signal processing of this document. log(' ...', nl=False) # When verbose output is requested, get the associated account. account = importer.account(filename) if verbose else None log(' OK', fg='green') log(f' {importer.name:}') log(f' {account:}', 1) if failfast and errors: break if errors: sys.exit(1) def _importer(importer): """Check that the passed instance implements the Importer interface. Wrap ImporterProtocol instances with Adapter as needed. """ if isinstance(importer, Importer): return importer if isinstance(importer, ImporterProtocol): warnings.warn('The beangulp.importer.ImporterProtocol interface for ' 'importers has been replaced by the beangulp.Importer ' 'interface and is therefore deprecated. Please update ' 'your importer {} to the new interface.'.format(importer), stacklevel=3) return Adapter(importer) raise TypeError(f'expected bengulp.Importer not {type(importer):}') class Ingest: def __init__(self, importers, hooks=None): self.importers = [_importer(i) for i in importers] self.hooks = list(hooks) if hooks is not None else [] while extract.find_duplicate_entries in self.hooks: self.hooks.remove(extract.find_duplicate_entries) warnings.warn('beangulp.extract.find_duplicate_entries has been removed ' 'from the import hooks. Deduplication is now integral part ' 'of the extract processing and can be customized by the ' 'importers. See beangulp.importer.Importer.', stacklevel=2) @click.group('beangulp') @click.version_option() @click.pass_context def cli(ctx): """Import data from and file away documents from financial institutions.""" ctx.obj = self cli.add_command(_archive) cli.add_command(_extract) cli.add_command(_identify) self.cli = cli def __call__(self): return self.cli() beangulp-0.2.0/beangulp/archive.py000066400000000000000000000046351474340320100171040ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import os import re import shutil from beangulp import utils from beangulp.exceptions import Error # Template for the documents archival name. FILENAME = '{date:%Y-%m-%d}.{name}' def filepath(importer, filepath: str) -> str: """Compute filing path for a document. The path mirrors the structure of the accounts associated to the documents and with a file name composed by the document date and document name returned by the importer. Args: importer: The importer instance to handle the document. filepath: Filesystem path to the document. Returns: Filing tree location where to store the document. Raises: beangulp.exceptions.Error: The canonical file name returned by the importer contains a path separator charachter or seems to contain a date. """ # Get the account corresponding to the file. account = importer.account(filepath) filename = importer.filename(filepath) or os.path.basename(filepath) date = importer.date(filepath) or utils.getmdate(filepath) # The returned filename cannot contain the file path separator character. if os.sep in filename: raise Error("The filename contains path separator character.") if re.match(r'\d\d\d\d-\d\d-\d\d\.', filename): raise Error("The filename contains what looks like a date.") # Prepend account directory and date prefix. filename = os.path.join(account.replace(':', os.sep), FILENAME.format(date=date, name=filename)) return filename def move(src: str, dst: str): """Move a file, potentially across devices. The destination direcory, and all intermediate path segments, are created if they do not exist. The move is performed with the shutil.move() function. See the documentation of this function for details of the semantic. Args: src: Filesystem path of the file to move. dst: Desitnation filesytem path. For the creation of the destination directory, this is assumed to be a file path and not a directory, namely the directory structure up to only path.dirname(dst) is created. """ # Create missing directories. os.makedirs(os.path.dirname(dst), exist_ok=True) # Copy the file to its new name: use shutil.move() instead of # os.rename() to support moving across filesystems. shutil.move(src, dst) beangulp-0.2.0/beangulp/archive_test.py000066400000000000000000000036521474340320100201410ustar00rootroot00000000000000import datetime import os import unittest from os import path from unittest import mock from beangulp import archive from beangulp import exceptions from beangulp import tests class TestFilepath(unittest.TestCase): def setUp(self): self.importer = tests.utils.Importer(None, 'Assets:Tests', None) def test_filepath(self): importer = mock.MagicMock(wraps=self.importer) importer.filename.return_value = 'foo.csv' filepath = archive.filepath(importer, path.abspath('test.pdf')) self.assertEqual(filepath, 'Assets/Tests/1970-01-01.foo.csv') def test_filepath_no_filename(self): filepath = archive.filepath(self.importer, path.abspath('test.pdf')) self.assertEqual(filepath, 'Assets/Tests/1970-01-01.test.pdf') def test_filepath_no_date(self): importer = mock.MagicMock(wraps=self.importer) importer.date.return_value = None with mock.patch('beangulp.archive.utils.getmdate', return_value=datetime.datetime.fromtimestamp( 0, datetime.timezone.utc)): filepath = archive.filepath(importer, path.abspath('test.pdf')) self.assertEqual(filepath, 'Assets/Tests/1970-01-01.test.pdf') def test_filepath_sep_in_name(self): importer = mock.MagicMock(wraps=self.importer) importer.filename.return_value = f'dir{os.sep:}name.pdf' with self.assertRaises(exceptions.Error) as ex: archive.filepath(importer, path.abspath('test.pdf')) self.assertRegex(ex.exception.message, r'contains path separator') def test_filepath_date_in_name(self): importer = mock.MagicMock(wraps=self.importer) importer.filename.return_value = '1970-01-03.name.pdf' with self.assertRaises(exceptions.Error) as ex: archive.filepath(importer, path.abspath('test.pdf')) self.assertRegex(ex.exception.message, r'contains [\w\s]+ date') beangulp-0.2.0/beangulp/cache.py000066400000000000000000000154031474340320100165210ustar00rootroot00000000000000"""A file wrapper which acts as a cache for on-demand evaluation of conversions. This object is used in lieu of a file in order to allow the various importers to reuse each others' conversion results. Converting file contents, e.g. PDF to text, can be expensive. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import codecs import functools import os import pickle import sys from contextlib import suppress from hashlib import sha1 from os import path import chardet from beangulp import mimetypes from beangulp import utils # NOTE: See get_file() at the end of this file to create instances of FileMemo. # Default location of cache directories. CACHEDIR = (path.expandvars('%LOCALAPPDATA%\\Beangulp') if sys.platform == 'win32' else path.expanduser('~/.cache/beangulp')) # Maximum number of bytes to read in order to detect the encoding of a file. HEAD_DETECT_MAX_BYTES = 128 * 1024 class _FileMemo: """A file memoizer which acts as a cache for on-demand evaluation of conversions. Attributes: name: A string, the name of the underlying file. """ def __init__(self, filename): self.name = filename # A cache of converter function to saved conversion value. self._cache = {} def __str__(self): return f'' def convert(self, converter_func): """Registers a callable used to convert the file contents. Args: converter_func: A callable which accepts a filename and produces some derived version of the file contents. Returns: A bytes object, with the contents of the entire file. """ try: result = self._cache[converter_func] except KeyError: # FIXME: Implement timing of conversions here. Store it for # reporting later. result = self._cache[converter_func] = converter_func(self.name) return result def mimetype(self): """Computes the MIME type of the file.""" return self.convert(mimetype) def head(self, num_bytes=8192, encoding=None): """An alias for reading just the first bytes of a file.""" return self.convert(head(num_bytes, encoding=encoding)) def contents(self): """An alias for reading the entire contents of the file.""" return self.convert(contents) def mimetype(filename): """A converter that computes the MIME type of the file. Returns: A converter function. """ mtype, _ = mimetypes.guess_type(filename, strict=False) return mtype def head(num_bytes=8192, encoding=None): """A converter that just reads the first bytes of a file. Note that the returned string may represent less bytes than specified if the encoded bytes read from the file terminate with an incomplete unicode character. This is likely to occur for variable width encodings suach as utf8. Args: num_bytes: The number of bytes to read. Returns: A converter function. """ def head_reader(filename): with open(filename, 'rb') as fd: data = fd.read(num_bytes) enc = encoding or chardet.detect(data)['encoding'] # A little trick to handle an encoded byte array that # terminates with an incomplete unicode character. decoder = codecs.iterdecode(iter([data]), enc) return next(decoder) return head_reader def contents(filename): """A converter that just reads the entire contents of a file. Args: num_bytes: The number of bytes to read. Returns: A converter function. """ # Attempt to detect the input encoding automatically, using chardet and a # decent amount of input. with open(filename, 'rb') as infile: rawdata = infile.read(HEAD_DETECT_MAX_BYTES) detected = chardet.detect(rawdata) encoding = detected['encoding'] # Ignore encoding errors for reading the contents because input files # routinely break this assumption. errors = 'ignore' with open(filename, encoding=encoding, errors=errors) as file: return file.read() def get_file(filename): """Create or reuse a globally registered instance of a FileMemo. Note: the FileMemo objects' lifetimes are reused for the duration of the process. This is usually the intended behavior. Always create them by calling this constructor. Args: filename: A path string, the absolute name of the file whose memo to create. Returns: A FileMemo instance. """ assert path.isabs(filename), ( "Path should be absolute in order to guarantee a single call.") return _CACHE[filename] _CACHE = utils.DefaultDictWithKey(_FileMemo) def cache(func=None, *, key=None): """Memoize the result of calling the given function to a disk pickle.""" def decorator(func): @functools.wraps(func) def wrapper(filename, *args, cache=None, **kwargs): # Compute the cache filename. input_key = key(filename) if key else filename name = sha1(pickle.dumps((input_key, args, kwargs))).hexdigest() + '.pickle' cache_fname = path.join(CACHEDIR, name) # We inspect the modified time of the input file and the cache. input_mtime = os.stat(filename).st_mtime_ns cache_mtime = 0 with suppress(FileNotFoundError): cache_mtime = os.stat(cache_fname).st_mtime_ns if cache is None: # Read from cache when a key function has been supplied and the # cache file exists or when the filename has been used to # compute the cache key and the cache entry modification time is # equal or later the input file modification time. cache = cache_mtime != 0 if key else cache_mtime >= input_mtime if cache: with open(cache_fname, 'rb') as f: return pickle.load(f) # Invoke the potentially expensive function. ret = func(filename, *args, **kwargs) # Ignore errors due to the CACHEDIR not being present. with suppress(FileNotFoundError): # To populate the cache atomically write the cache entry in a # temporary file and move it to the right place with the # complete content and the right modification time. cache_temp = cache_fname + '~' with open(cache_temp, 'wb') as f: pickle.dump(ret, f) os.utime(cache_temp, ns=(input_mtime, input_mtime)) os.replace(cache_temp, cache_fname) return ret return wrapper if func is None: return decorator return decorator(func) beangulp-0.2.0/beangulp/cache_test.py000066400000000000000000000141541474340320100175620ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import os import shutil import tempfile import time import unittest from unittest import mock from beangulp import cache from beangulp import utils class TestFileMemo(unittest.TestCase): def test_cache(self): with tempfile.NamedTemporaryFile() as tmpfile: shutil.copy(__file__, tmpfile.name) wrap = cache._FileMemo(tmpfile.name) # Check attributes. self.assertEqual(tmpfile.name, wrap.name) # Check that caching works. converter = mock.MagicMock(return_value='abc') self.assertEqual('abc', wrap.convert(converter)) self.assertEqual('abc', wrap.convert(converter)) self.assertEqual('abc', wrap.convert(converter)) self.assertEqual(1, converter.call_count) def test_cache_head_and_contents(self): with tempfile.NamedTemporaryFile(suffix='.py') as tmpfile: shutil.copy(__file__, tmpfile.name) wrap = cache._FileMemo(tmpfile.name) contents = wrap.convert(cache.contents) self.assertIsInstance(contents, str) self.assertGreater(len(contents), 128) contents2 = wrap.contents() self.assertEqual(contents, contents2) head = wrap.convert(cache.head(128)) self.assertIsInstance(head, str) self.assertEqual(128, len(head)) mimetype = wrap.convert(cache.mimetype) self.assertIn(mimetype, {'text/plain', 'text/x-python', 'text/x-script.python', 'text/c++'}) def test_cache_head_obeys_explict_utf8_encoding_avoids_chardet_exception(self): data = b'asciiHeader1,\xf0\x9f\x8d\x8fHeader1,asciiHeader2' with mock.patch('builtins.open', mock.mock_open(read_data=data)): string = cache._FileMemo('filepath').head(encoding='utf-8') self.assertEqual(string, data.decode('utf8')) def test_cache_head_encoding(self): data = b'asciiHeader1,\xf0\x9f\x8d\x8fHeader1,asciiHeader2' # The 15th bytes is in the middle of the unicode character. num_bytes = 15 with mock.patch('builtins.open', mock.mock_open(read_data=data)): string = cache._FileMemo('filepath').head(num_bytes, encoding='utf-8') self.assertEqual(string, 'asciiHeader1,') class CacheTestCase(unittest.TestCase): def setUp(self): self.tempdir = tempfile.mkdtemp() cache.CACHEDIR = os.path.join(self.tempdir, 'cache') os.mkdir(cache.CACHEDIR) self.filename = os.path.join(self.tempdir, 'test.txt') with open(self.filename, 'w'): pass def tearDown(self): shutil.rmtree(self.tempdir) def test_no_cache(self): counter = 0 def func(filename): nonlocal counter counter +=1 return counter r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename) self.assertEqual(r, 2) def test_cache(self): counter = 0 @cache.cache def func(filename): nonlocal counter counter +=1 return counter r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename) self.assertEqual(r, 1) def test_cache_expire_mtime(self): counter = 0 @cache.cache def func(filename): nonlocal counter counter +=1 return counter r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename) self.assertEqual(r, 1) t = time.time() + 2.0 os.utime(self.filename, (t, t)) r = func(self.filename) self.assertEqual(r, 2) r = func(self.filename) self.assertEqual(r, 2) def test_cache_expire_args(self): counter = 0 @cache.cache def func(filename, arg): nonlocal counter counter +=1 return counter r = func(self.filename, 1) self.assertEqual(r, 1) r = func(self.filename, 1) self.assertEqual(r, 1) r = func(self.filename, 2) self.assertEqual(r, 2) def test_cache_expire_override(self): counter = 0 @cache.cache def func(filename): nonlocal counter counter +=1 return counter r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename, cache=False) self.assertEqual(r, 2) r = func(self.filename, cache=True) self.assertEqual(r, 2) t = time.time() + 2.0 os.utime(self.filename, (t, t)) r = func(self.filename, cache=True) self.assertEqual(r, 2) def test_cache_reset_mtime(self): counter = 0 @cache.cache def func(filename): nonlocal counter counter +=1 return counter r = func(self.filename) self.assertEqual(r, 1) t = os.stat(self.filename).st_mtime_ns with open(self.filename, 'w') as f: f.write('baz') os.utime(self.filename, ns=(t, t)) r = func(self.filename) self.assertEqual(r, 1) def test_cache_key_sha1(self): counter = 0 @cache.cache(key=utils.sha1sum) def func(filename): nonlocal counter counter +=1 return counter with open(self.filename, 'w') as f: f.write('test') r = func(self.filename) self.assertEqual(r, 1) r = func(self.filename) self.assertEqual(r, 1) t = time.time() + 2.0 os.utime(self.filename, (t, t)) r = func(self.filename) self.assertEqual(r, 1) t = os.stat(self.filename).st_mtime_ns with open(self.filename, 'w') as f: f.write('baz') os.utime(self.filename, ns=(t, t)) r = func(self.filename) self.assertEqual(r, 2) beangulp-0.2.0/beangulp/csv_utils.py000066400000000000000000000132601474340320100174700ustar00rootroot00000000000000""" Utilities for reading and writing CSV files. """ __copyright__ = "Copyright (C) 2013-2014, 2016 Martin Blais" __license__ = "GNU GPLv2" import itertools import collections import csv import re import io import textwrap def as_rows(string): """Split a string as rows of a CSV file. Args: string: A string to be split, the contents of a CSV file. Returns: Lists of lists of strings. """ return list(csv.reader(io.StringIO(textwrap.dedent(string)))) def csv_clean_header(header_row): """Create a new class for the following rows from the header line. This both normalizes the header line and assign Args: header_row: A list of strings, the row with header titles. Returns: A list of strings, with possibly modified (cleaned) row titles, of the same lengths. """ fieldnames = [] for index, column in enumerate(header_row): field = column.lower() field = re.sub(r'\bp/l\b', 'pnl', field) field = re.sub(r'[^a-z0-9]', '_', field) field = field.strip(' _') field = re.sub(r'__+', '_', field) if not field: field = f'col{index}' assert field not in fieldnames, field fieldnames.append(field) return fieldnames def csv_dict_reader(fileobj, **kw): """Read a CSV file yielding normalized dictionary fields. This is basically an alternative constructor for csv.DictReader that normalized the field names. Args: fileobj: A file object to be read. **kw: Optional arguments forwarded to csv.DictReader. Returns: A csv.DictReader object. """ reader = csv.DictReader(fileobj, **kw) reader.fieldnames = csv_clean_header(reader.fieldnames) return reader def csv_tuple_reader(fileobj, **kw): """Read a CSV file yielding namedtuple instances. The CSV file must have a header line. Args: fileobj: A file object to be read. **kw: Optional arguments forwarded to csv.DictReader. Yields: Nametuple instances, one for each row. """ reader = csv.reader(fileobj, **kw) ireader = iter(reader) fieldnames = csv_clean_header(next(ireader)) Tuple = collections.namedtuple('Row', fieldnames) for row in ireader: try: yield Tuple(*row) except TypeError: # If there's an error, it's usually from a line that has a 'END OF # LINE' marker at the end, or some comment line. assert len(row) in (0, 1) def csv_split_sections(rows): """Given rows, split them in at empty lines. This is useful for structured CSV files with multiple sections. Args: rows: A list of rows, which are themselves lists of strings. Returns: A list of sections, which are lists of rows, which are lists of strings. """ sections = [] current_section = [] for row in rows: if any(cell.strip() for cell in row): # Append to existing section. current_section.append(row) else: # Row is empty, end section. sections.append(current_section) current_section = [] if current_section: sections.append(current_section) return sections def csv_split_sections_with_titles(rows): """Given a list of rows, split their sections. If the sections have single column titles, consume those lines as their names and return a mapping of section names. This is useful for CSV files with multiple sections, where the separator is a title. We use this to separate the multiple tables within the CSV files. Args: rows: A list of rows (list-of-strings). Returns: A list of lists of rows (list-of-strings). """ sections_map = {} for index, section in enumerate(csv_split_sections(rows)): # Skip too short sections, cannot possibly be a title. if len(section) < 2: continue if len(section[0]) == 1 and len(section[1]) != 1: name = section[0][0] section = section[1:] else: name = f'Section {index}' sections_map[name] = section return sections_map def iter_sections(fileobj, separating_predicate=None): """For a given file object, yield file-like objects for each of the sections contained therein. A section is defined as a list of lines that don't match the predicate. For example, if you want to split by empty lines, provide a predicate that will be true given an empty line, which will cause a new section to be begun. Args: fileobj: A file object to read from. separating_predicate: A boolean predicate that is true on separating lines. Yields: A list of lines that you can use to iterate. """ if separating_predicate is None: separating_predicate = lambda line: bool(line.strip()) lineiter = iter(fileobj) for line in lineiter: if separating_predicate(line): iterator = itertools.chain((line,), iter_until_empty(lineiter, separating_predicate)) yield iterator for _ in iterator: pass def iter_until_empty(iterator, separating_predicate=None): """An iterator of lines that will stop at the first empty line. Args: iterator: An iterator of lines. separating_predicate: A boolean predicate that is true on separating lines. Yields: Non-empty lines. EOF when we hit an empty line. """ if separating_predicate is None: separating_predicate = lambda line: bool(line.strip()) for line in iterator: if not separating_predicate(line): break yield line beangulp-0.2.0/beangulp/csv_utils_test.py000066400000000000000000000170321474340320100205300ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import unittest import io import tempfile import textwrap from beangulp import csv_utils class TestCSVUtils(unittest.TestCase): def test_csv_clean_header(self): self.assertEqual(['date', 'pnl', 'balance'], csv_utils.csv_clean_header('date,p/l,balance'.split(','))) self.assertEqual(['date', 'day_s_range', 'balance'], csv_utils.csv_clean_header("date,day's range ,balance".split(','))) self.assertEqual(['date', 'col1', 'balance'], csv_utils.csv_clean_header("date,,balance".split(','))) def test_csv_dict_reader(self): with tempfile.NamedTemporaryFile('w') as tmpfile: tmpfile.write(textwrap.dedent( """\ First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada """)) tmpfile.flush() with open(tmpfile.name) as infile: reader = csv_utils.csv_dict_reader(infile, skipinitialspace=True) self.assertTrue(isinstance(reader, object)) self.assertEqual([ {'first_name': 'Caroline', 'last_name': 'Chang', 'city': 'Sydney', 'country': 'Australia'}, {'first_name': 'Martin', 'last_name': 'Blais', 'city': 'Vancouver', 'country': 'Canada'} ], list(reader)) def test_csv_tuple_reader(self): with tempfile.NamedTemporaryFile('w') as tmpfile: tmpfile.write(textwrap.dedent( """\ First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada """)) tmpfile.flush() with open(tmpfile.name) as infile: reader = csv_utils.csv_tuple_reader(infile, skipinitialspace=True) self.assertTrue(isinstance(reader, object)) rows = list(reader) first_row = rows[0] self.assertTrue(isinstance(first_row, tuple)) self.assertTrue(first_row.first_name) self.assertTrue(first_row.last_name) self.assertTrue(first_row.city) self.assertTrue(first_row.country) self.assertEqual(2, len(rows)) def test_csv_split_sections(self): rows = csv_utils.as_rows("""\ Names: First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada Ages: Last Name, SSN, Age Blais, 123-45-6789, 41 Chang, 987-65-4321, 32 """) sections = csv_utils.csv_split_sections(rows) self.assertEqual( [[['Names:'], ['First Name', ' Last Name', ' City', ' Country'], ['Caroline', ' Chang', ' Sydney', ' Australia'], ['Martin', ' Blais', ' Vancouver', ' Canada']], [['Ages:'], ['Last Name', ' SSN', ' Age'], ['Blais', ' 123-45-6789', ' 41'], ['Chang', ' 987-65-4321', ' 32']]], sections) rows = csv_utils.as_rows("""\ Names: First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada """) sections = csv_utils.csv_split_sections(rows) self.assertEqual( [[['Names:'], ['First Name', ' Last Name', ' City', ' Country'], ['Caroline', ' Chang', ' Sydney', ' Australia'], ['Martin', ' Blais', ' Vancouver', ' Canada']]], sections) rows = csv_utils.as_rows("") sections = csv_utils.csv_split_sections(rows) self.assertEqual([], sections) def test_csv_split_sections_with_titles(self): rows = csv_utils.as_rows("""\ Names: First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada Ages: Last Name, SSN, Age Blais, 123-45-6789, 41 Chang, 987-65-4321, 32 """) sections = csv_utils.csv_split_sections_with_titles(rows) self.assertEqual( {'Names:': [['First Name', ' Last Name', ' City', ' Country'], ['Caroline', ' Chang', ' Sydney', ' Australia'], ['Martin', ' Blais', ' Vancouver', ' Canada']], 'Ages:': [['Last Name', ' SSN', ' Age'], ['Blais', ' 123-45-6789', ' 41'], ['Chang', ' 987-65-4321', ' 32']]}, sections) rows = csv_utils.as_rows("""\ Names: First Name, Last Name, City, Country Caroline, Chang, Sydney, Australia Martin, Blais, Vancouver, Canada """) sections = csv_utils.csv_split_sections_with_titles(rows) self.assertEqual( {'Names:': [['First Name', ' Last Name', ' City', ' Country'], ['Caroline', ' Chang', ' Sydney', ' Australia'], ['Martin', ' Blais', ' Vancouver', ' Canada']]}, sections) rows = csv_utils.as_rows("") sections = csv_utils.csv_split_sections_with_titles(rows) self.assertEqual({}, sections) def linearize(iterator, joiner=list): """Consume a section iterator. Args: iterator: An iterator of iterators. joiner: A callable to apply to the sub-iterators. Returns: A list of return values from joiner. """ return list(map(list, iterator)) class TestLineUtils(unittest.TestCase): def test_iter_until_empty(self): iterator = iter(['a', 'b', '', 'c']) prefix = list(csv_utils.iter_until_empty(iterator)) self.assertEqual(['a', 'b'], prefix) self.assertEqual('c', next(iterator)) iterator = iter(['a', '', '', 'c']) prefix = list(csv_utils.iter_until_empty(iterator)) self.assertEqual(['a'], prefix) prefix = list(csv_utils.iter_until_empty(iterator)) self.assertEqual([], prefix) self.assertEqual('c', next(iterator)) def test_iter_section(self): # Test with empty string. sio = io.StringIO("") self.assertEqual([], linearize(csv_utils.iter_sections(sio))) # Test with just empty lines, and a final empty line. sio = io.StringIO("\n\n \n \n\n ") self.assertEqual([], linearize(csv_utils.iter_sections(sio))) # Test with a simple non-empty line. sio = io.StringIO("\n\n\n\n\nWORD\n\n\n") self.assertEqual([['WORD\n']], linearize(csv_utils.iter_sections(sio))) # Test with a simple non-empty line, at the end. sio = io.StringIO("\n\n\n\n\nWORD\n") self.assertEqual([['WORD\n']], linearize(csv_utils.iter_sections(sio))) # Test with a simple non-empty line, at the very end, without a newline. sio = io.StringIO("\n\n\n\n\nWORD") self.assertEqual([['WORD']], linearize(csv_utils.iter_sections(sio))) # Test with output that looks regular. sio = io.StringIO("Title1\nA,B,C\nD,E,F\n\n\n\nTitle2\nG,H\nI,J\n,K,L\n\n\n") expected = [['Title1\n', 'A,B,C\n', 'D,E,F\n'], ['Title2\n', 'G,H\n', 'I,J\n', ',K,L\n']] self.assertEqual(expected, linearize(csv_utils.iter_sections(sio))) if __name__ == '__main__': unittest.main() beangulp-0.2.0/beangulp/date_utils.py000066400000000000000000000011441474340320100176100ustar00rootroot00000000000000import dateutil.parser def parse_date(string, parse_kwargs_dict=None): """Parse arbitrary strings to dates. This function is intended to support liberal inputs, so that we can use it in accepting user-specified dates on command-line scripts. Args: string: A string to parse. parse_kwargs_dict: Dict of kwargs to pass to dateutil parser. Returns: A datetime.date object. """ # At the moment, rely on the most excellent dateutil. if parse_kwargs_dict is None: parse_kwargs_dict = {} return dateutil.parser.parse(string, **parse_kwargs_dict).date() beangulp-0.2.0/beangulp/date_utils_test.py000066400000000000000000000003441474340320100206500ustar00rootroot00000000000000import datetime import unittest from beangulp import date_utils class TestDateUtils(unittest.TestCase): def test_parse_date(self): self.assertEqual(datetime.date(2021, 7, 4), date_utils.parse_date('2021-07-04')) beangulp-0.2.0/beangulp/exceptions.py000066400000000000000000000027051474340320100176400ustar00rootroot00000000000000import contextlib import textwrap import traceback class Error(RuntimeError): def __init__(self, message, *args): self.message = message self.args = args def __str__(self): return '\n'.join((*self.args, self.message)) class ExceptionsTrap(contextlib.AbstractContextManager): """A context manager to log exceptions. Works similarly to contextlib.suppress() but logs the exceptions instead than simply discarding them. This is used to shorten and unify exception handling in the command line wrappers. The format of the reporting is specific to the Beangulp command line interface. """ def __init__(self, func): self.log = func self.errors = 0 def __exit__(self, exctype, excinst, exctb): if exctype is None: return True self.errors += 1 self.log(' ERROR', fg='red') if issubclass(exctype, Error): # Beangulp validation error. self.log(textwrap.indent(str(excinst), ' ')) return True if issubclass(exctype, Exception): # Unexpected exception. self.log(' Exception in importer code.') exc = ''.join(traceback.format_exception(exctype, excinst, exctb)) self.log(textwrap.indent(exc, ' ').rstrip()) return True return False def __bool__(self): """Return True if any error occurred.""" return self.errors != 0 beangulp-0.2.0/beangulp/extract.py000066400000000000000000000202701474340320100171260ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016-2017 Martin Blais" __license__ = "GNU GPLv2" import bisect import datetime import operator import textwrap import warnings from typing import Callable from beancount.core import data from beancount.parser import printer from beangulp import similar # Header for the file where the extracted entries are written. HEADER = ';; -*- mode: beancount -*-\n' # Format for the section titles separating entries extracted from # different documents. This is used as a format sting passing the # document filesystem path as argument. SECTION = '**** {}' # Metadata field that indicates the entry is a likely duplicate. DUPLICATE = '__duplicate__' def extract_from_file(importer, filename, existing_entries): """Import entries from a document. Args: importer: The importer instance to handle the document. filename: Filesystem path to the document. existing_entries: Existing entries. Returns: The list of imported entries. """ entries = importer.extract(filename, existing_entries) if not entries: return [] # Sort the newly imported entries. importer.sort(entries) # Ensure that the entries are typed correctly. for entry in entries: data.sanity_check_types(entry) return entries def sort_extracted_entries(extracted): """Sort the extraxted entries. Sort extracged entries, grouped by source document, in the order in which they will be used in deduplication and in which they will be serialized to file. Args: extracted: List of (filepath, entries, account, importer) tuples where entries is the list of entries extracted from the document at filepath by importer. """ # The entries are sorted on a key composed by (max-date, account, # min-date, filename) where max-date and min-date are the latest # and earliest date appearing in the entries list. This should # place entries from documents produced earlier in time at before # ones coming from documents produced later. # # Most imports have a balance statement at the end with a date # that is one day later than the reporting period (balance # statement are effective at the beginning of the day). Thus # using the end date should be more predictable than sorting on # the earliest entry. # # This diagram, where the bars represents the time span covered by # contained entries, represent the sort order we want to obtain: # # Assets:Ali (-----) # Assets:Ali (=====--------) # Assets:Bob (------------) # Assets:Bob (===----) # Assets:Ali (--------------) # Assets:Bob (====-----------) # # The sections marked with = represent the time spans in which # duplicated entries could be present. We want entries form # documents produced earlier in time to take precedence over # entries from documents produced later in time. def key(element): filename, entries, account, importer = element dates = [entry.date for entry in entries] # Sort documents that do not contain any entry last. max_date = min_date = datetime.date(9999, 1, 1) if dates: max_date, min_date = max(dates), min(dates) return max_date, account, min_date, filename extracted.sort(key=key) def find_duplicate_entries(extracted, existing): """Flag potentially duplicate entries. Args: extracted: List of (filepath, entries, account, importer) tuples where entries is the list of entries extracted from the document at filepath by importer. existing: Existing entries. Returns: A copy of the list of new entries with the potentially duplicate entries marked setting the "__duplicate__" metadata field to True. """ # This function is kept only for backwards compatibility. warnings.warn('The find_duplicate_entries() function is kept only for ' 'backwards compatibility with import scripts that explicitly ' 'added it to the import hooks. It does not conform to the ' 'current way of implementing deduplication and it is not to ' 'be used.', stacklevel=2) ret = [] for filepath, entries, account, importer in extracted: # Sort the existing entries by date: find_similar_entries() # uses bisection to reduce the list of existing entries to the # set in a narrow date interval around the date of each entry # in the set it is comparing against. existing.sort(key=operator.attrgetter('date')) # Find duplicates. pairs = similar.find_similar_entries(entries, existing) # We could do something smarter than throwing away the # information about which entry is the source of the possible # duplication. duplicates = { id(duplicate) for duplicate, source in pairs } marked = [] for entry in entries: if id(entry) in duplicates: meta = entry.meta.copy() meta[DUPLICATE] = True entry = entry._replace(meta=meta) marked.append(entry) ret.append((filepath, marked, account, importer)) # Append the current batch of extracted entries to the # existing entries. This allows to deduplicate entries in the # current extraction run. existing.extend(marked) return ret def mark_duplicate_entries( entries: data.Entries, existing: data.Entries, window: datetime.timedelta, compare: Callable[[data.Directive, data.Directive], bool]) -> None: """Mark duplicate entries. Compare newly extracted entries to the existing entries. Only existing entries dated within the given time window around the date of the each existing entry. Entries that are determined to be duplicates of existing entries are marked setting the "__duplicate__" metadata field. Args: entries: Entries to be deduplicated. existing: Existing entries. window: Time window in which entries are compared. compare: Entry comparison function. """ # The use of bisection to identify the entries in the existing # list that have dates within a given window around the date # of each newly extracted entry requires the existing entries # to be sorted by date. existing.sort(key=operator.attrgetter('date')) dates = [entry.date for entry in existing] def entries_date_window_iterator(date): lo = bisect.bisect_left(dates, date - window) hi = bisect.bisect_right(dates, date + window) for i in range(lo, hi): yield existing[i] for entry in entries: for target in entries_date_window_iterator(entry.date): if compare(entry, target): entry.meta[DUPLICATE] = target def print_extracted_entries(extracted, output): """Print extracted entries. Entries marked as duplicates are printed as comments. Args: extracted: List of (filepath, entries, account, importer) tuples where entries is the list of entries extracted from the document at filepath by importer. output: A file object to write to. The object needs to implement a write() method that accepts an unicode string. """ if extracted and HEADER: output.write(HEADER + '\n') for filepath, entries, account, importer in extracted: output.write(SECTION.format(filepath) + '\n\n') for entry in entries: duplicate = entry.meta.pop(DUPLICATE, False) string = printer.format_entry(entry) # If the entry is a duplicate, comment it out and report # of which other entry this is a duplicate. if duplicate: if isinstance(duplicate, type(entry)): filename = duplicate.meta.get('filename') lineno = duplicate.meta.get('lineno') if filename and lineno: output.write(f'; duplicate of {filename}:{lineno}\n') string = textwrap.indent(string, '; ') output.write(string) output.write('\n') output.write('\n') beangulp-0.2.0/beangulp/extract_test.py000066400000000000000000000074241474340320100201730ustar00rootroot00000000000000import io import textwrap import unittest from datetime import timedelta from os import path from unittest import mock from beancount.parser import parser from beangulp import extract from beangulp import similar from beangulp import tests class TestExtract(unittest.TestCase): def setUp(self): self.importer = tests.utils.Importer(None, 'Assets:Tests', None) def test_extract_from_file_no_entries(self): entries = extract.extract_from_file(self.importer, path.abspath('test.csv'), []) self.assertEqual(entries, []) def test_extract_from_file(self): entries, errors, options = parser.parse_string(textwrap.dedent(''' 1970-01-03 * "Test" Assets:Tests 1.00 USD 1970-01-01 * "Test" Assets:Tests 1.00 USD 1970-01-02 * "Test" Assets:Tests 1.00 USD ''')) importer = mock.MagicMock(wraps=self.importer) importer.extract.return_value = entries entries = extract.extract_from_file(importer, path.abspath('test.csv'), []) dates = [entry.date for entry in entries] self.assertSequenceEqual(dates, sorted(dates)) def test_extract_from_file_ensure_sanity(self): entries, errors, options = parser.parse_string(''' 1970-01-01 * "Test" Assets:Tests 1.00 USD ''') # Break something. entries[-1] = entries[-1]._replace(narration=42) importer = mock.MagicMock(wraps=self.importer) importer.extract.return_value = entries with self.assertRaises(AssertionError): extract.extract_from_file(importer, path.abspath('test.csv'), []) class TestDuplicates(unittest.TestCase): def test_mark_duplicate_entries(self): entries, error, options = parser.parse_string(textwrap.dedent(''' 1970-01-01 * "Test" Assets:Tests 10.00 USD 1970-01-02 * "Test" Assets:Tests 20.00 USD ''')) compare = similar.heuristic_comparator() extract.mark_duplicate_entries(entries, entries[:1], timedelta(days=2), compare) self.assertTrue(entries[0].meta[extract.DUPLICATE]) self.assertNotIn(extract.DUPLICATE, entries[1].meta) class TestPrint(unittest.TestCase): def test_print_extracted_entries(self): entries, error, options = parser.parse_string(textwrap.dedent(''' 1970-01-01 * "Test" Assets:Tests 10.00 USD''')) extracted = [ ('/path/to/test.csv', entries, None, None), ('/path/to/empty.pdf', [], None, None), ] output = io.StringIO() extract.print_extracted_entries(extracted, output) self.assertEqual(output.getvalue(), textwrap.dedent('''\ ;; -*- mode: beancount -*- **** /path/to/test.csv 1970-01-01 * "Test" Assets:Tests 10.00 USD **** /path/to/empty.pdf ''')) def test_print_extracted_entries_duplictes(self): entries, error, options = parser.parse_string(textwrap.dedent(''' 1970-01-01 * "Test" Assets:Tests 10.00 USD 1970-01-01 * "Test" Assets:Tests 10.00 USD ''')) # Mark the second entry as duplicate entries[1].meta[extract.DUPLICATE] = True extracted = [ ('/path/to/test.csv', entries, None, None), ] output = io.StringIO() extract.print_extracted_entries(extracted, output) self.assertEqual(output.getvalue(), textwrap.dedent('''\ ;; -*- mode: beancount -*- **** /path/to/test.csv 1970-01-01 * "Test" Assets:Tests 10.00 USD ; 1970-01-01 * "Test" ; Assets:Tests 10.00 USD ''')) beangulp-0.2.0/beangulp/file_type.py000066400000000000000000000025211474340320100174330ustar00rootroot00000000000000"""Code that guesses a MIME type for a filename. This attempts to identify the mime-type of a file using the built-in mimetypes library, augmented with MIME types commonly used in financial downloads. If this does not produce any match it falls back to MIME type sniffing using ``python-magic``, if available. This module is deprecated. Please use ``beancount.mimetypes`` instead. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import warnings from beangulp import mimetypes # python-magic is an optional dependency. try: import magic except (ImportError, OSError): magic = None def guess_file_type(filename): """Attempt to guess the type of the input file. Args: filename: A string, the name of the file to guess the type for. Returns: A suitable mimetype string, or None if we could not guess. """ warnings.warn('beangulp.file_type.guess_file_type() is deprecated. ' 'Use the beangulp.mimetypes module instead.', DeprecationWarning, stacklevel=2) filetype, encoding = mimetypes.guess_type(filename, strict=False) if filetype: return filetype if magic: filetype = magic.from_file(filename, mime=True) if isinstance(filetype, bytes): filetype = filetype.decode('utf8') return filetype beangulp-0.2.0/beangulp/file_type_test.py000066400000000000000000000063521474340320100205000ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import os from os import path import unittest import warnings from beangulp import file_type class TestFileType(unittest.TestCase): DATA_DIR = path.join(path.dirname(__file__), 'file_type_testdata') def check_mime_type(self, example_file, expected_mime_types): if not isinstance(expected_mime_types, list): expected_mime_types = [expected_mime_types] with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) mime_type = file_type.guess_file_type( os.path.realpath(path.join(self.DATA_DIR, example_file))) self.assertIn(mime_type, expected_mime_types) def test_csv(self): self.check_mime_type('example.csv', ['text/csv', 'text/x-comma-separated-values']) def test_xls(self): self.check_mime_type('example.xls', ['application/excel', 'application/vnd.ms-excel']) def test_ods(self): self.check_mime_type('example.ods', 'application/vnd.oasis.opendocument.spreadsheet') def test_ps(self): self.check_mime_type('example.ps', 'application/postscript') def test_pdf(self): self.check_mime_type('example.pdf', 'application/pdf') def test_ofx(self): self.check_mime_type('example.ofx', 'application/x-ofx') def test_qbo(self): self.check_mime_type('example.qbo', 'application/vnd.intu.qbo') def test_qfx(self): self.check_mime_type('example.qfx', ['application/x-ofx', 'application/vnd.intu.qfx']) def test_py(self): self.check_mime_type('example.py', 'text/x-python') def test_sh(self): self.check_mime_type('example.sh', ['text/x-sh', 'application/x-shellscript', 'application/x-sh']) def test_jpg(self): self.check_mime_type('example.jpg', 'image/jpeg') def test_txt(self): self.check_mime_type('example.txt', 'text/plain') def test_org(self): self.check_mime_type('example.org', ['text/plain', 'application/vnd.lotus-organizer']) def test_xml(self): self.check_mime_type('example.xml', ['text/xml', 'application/xml']) def test_html(self): self.check_mime_type('example.html', 'text/html') def test_xhtml(self): self.check_mime_type('example.xhtml', ['application/xhtml+xml', 'text/html']) def test_zip(self): self.check_mime_type('example.zip', ['application/zip', 'application/x-zip-compressed']) @unittest.skipIf(not file_type.magic, 'python-magic is not installed') def test_gz(self): self.check_mime_type('example.gz', ['application/gzip', 'application/x-gzip']) @unittest.skipIf(not file_type.magic, 'python-magic is not installed') def test_bz2(self): self.check_mime_type('example.bz2', 'application/x-bzip2') beangulp-0.2.0/beangulp/file_type_testdata/000077500000000000000000000000001474340320100207525ustar00rootroot00000000000000beangulp-0.2.0/beangulp/file_type_testdata/example.bz2000066400000000000000000000003451474340320100230260ustar00rootroot00000000000000BZh91AY&SY-4߀@@?,$X?ޠ0zdh@ j4Qhޤ2  &[FFW#]1i\Nw{oU# guV1Fh!ZI?Ki˔Bk8B(Sh."SHP2`H狍k]ԇ$RvTp0 $rE8P-beangulp-0.2.0/beangulp/file_type_testdata/example.csv000066400000000000000000000011451474340320100231230ustar00rootroot00000000000000DATE,TRANSACTION ID,DESCRIPTION,QUANTITY,SYMBOL,PRICE,COMMISSION,AMOUNT,NET CASH BALANCE,REG FEE,SHORT-TERM RDM FEE,FUND REDEMPTION FEE, DEFERRED SALES CHARGE 07/02/2013,10223506553,ORDINARY DIVIDEND (HDV),,HDV,,,31.04,31.04,,,, 07/02/2013,10224851005,MONEY MARKET PURCHASE,,,,,-31.04,0.00,,,, 07/02/2013,10224851017,MONEY MARKET PURCHASE (MMDA10),31.04,MMDA10,,,0.00,0.00,,,, 09/30/2013,10561187188,ORDINARY DIVIDEND (HDV),,HDV,,,31.19,31.19,,,, 09/30/2013,10563719172,MONEY MARKET PURCHASE,,,,,-31.19,0.00,,,, 09/30/2013,10563719198,MONEY MARKET PURCHASE (MMDA10),31.19,MMDA10,,,0.00,0.00,,,, ***END OF FILE*** beangulp-0.2.0/beangulp/file_type_testdata/example.gz000066400000000000000000000003401474340320100227440ustar00rootroot00000000000000e7SexampleKn0 D:\/ xQ] h(!Qr.О ܽ3[8L:t81F!5L\0jF:$c|Šz:1/Q&$ݤinb-t]<.>Ţ&=Ǭ _FV˦>ƅ%p' (%~n>HR ^.@=KmW5;~&beangulp-0.2.0/beangulp/file_type_testdata/example.html000066400000000000000000000005641474340320100233000ustar00rootroot00000000000000 Misc
beangulp-0.2.0/beangulp/file_type_testdata/example.jpg000066400000000000000000000025311474340320100231100ustar00rootroot00000000000000JFIFHHJCopyright (C) 2004 Martin Blais . All Rights Reserved.C!7$!!D03(7PFTSOFMLXclX^x_LMnpxVjC!!A$$A[M[@@ !$K 9dˀTB&!Ώ@ju9t;ƛ`4P.9ԚSZxVvbUxCATBU 1!2A"c\a7n*Naqs[/EÝKV A\v1}` c#2yʵDB0 5f2?]kX@ 0?VXod:0 @?N" !q"QA?4~IZq+{&۴ Bm1!AQ1a?!.,rslOw\}dadR!$>Jad3@0G̭0Wp "DɤܩDz.?VthL.beangulp-0.2.0/beangulp/file_type_testdata/example.ods000066400000000000000000000160551474340320100231230ustar00rootroot00000000000000PK~Dl9..mimetypeapplication/vnd.oasis.opendocument.spreadsheetPK~D8Thumbnails/thumbnail.pngPNG  IHDR[{IDATxKVeobM)bZAWK[4J- ]F)tD(P ".]dDMȠ0A ~P3w_8/g8 /+7l\d8X^;_>qǶMW{2kÅ/GS9w_\ڸsCӓ7]wϝF[uyM+||eW^<˻Ompn0XU3q'oGߵ3?ɫjMWN8q.[1aoa?/vX..""IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!tC$"IH:D!t~e)nFIENDB`PK~D settings.xmlYMs0W0I:mh$IqZ$__ڍVKzVoʧZytp}}f0xs9ы-A)=Er.loE;H$'HGyel'5=gRcqć(vNG9]56 % 2gRcN~[[Y<n dm Hb7?]A>!|Qn๿ sV;b=q  A_T4 -aQ5u5d xGpwgA=Cd@5P"r䒇բj8G+%1 dlFDub{XbL kJ9a"S%*Sл§2h>=n`_ DHz @|ݗ1fcy)ˁfȤ Œۈp"7!1faRw#K5n4J/w~Tk-r/6h uC $;qDTu3dLVD F{'d@Ot@D'!̈́d5'?\>Jc,CI "R7ah14]0o5.a^d&RqUvm -fb?#('b]QIFUacXeT\}My a(Bj*ե-7PKx>:PK~D content.xmlWM6 W.Л|3nEaҙ-zU$:QW I__qj`{EQY &&"A΍a|<'w?mLI0Ԑ{Mwܥw6O sҥ9RSS@ޱֹ5^cJg1[XVM%,5gf* V]g%}RZUռZύ= `Ҫ%8!vX M{KQ /L6}nzmGzB܅51$T?f~x5(  Hv_=Sf\4c) 2i `*KPQ JvsءH#p[}.tměʝ[{ >8mn_%"o! dmܛg{нM~acZS Džutn(8N ԿÿrrwG/0K8{4?e.䨦4ѱkI9X7>Y+>xRInj%Yfҽ]^I|u?3͎iXU #bi .Sx"1Ψ,)H+YTTHW(viED%~T:co:#s wM~2 mlu8r]h~Jc z&7G"S%)Ųp8ˆކ1aKdHuqԕ*OGgPK\.X PK~D styles.xmlYmo6_a(C%;VEmQtݾ3%sD_#)Ғe[4H#玼s}{lG (" -/ѓv5OSu*'BR30.N(K*Ή\xKR8u6Kl79۲wW6u"~ƂMO5>HRbX3F7Vr~sjYO8̠8$d/B͉SilRQwDL+܉j)lW4GMV~ٵdXL3ne2=U.mv O70i~y}+O]Kc[RłiM{ι a7tQtz? hQxYyh[@dSޡǡ %I_vխQճ$-dѺ_ԼF Q1Vd*{PAfl:WR^~x9l(U`Rʡ&8&(!1k{}ngbOMS=2Rn JBTp8wl/ȟj.dic)"pzͷT^"o)]=n[ =!)Xx85G(&^b3-*P;hKPpȥq~J\mP Y3#,(? 8Oq~]b1rwA[zODG͵-umoj6 @)"%4JfG\ oEt.rqw㔨Hqش#氍m~\ 昡Aޖl&xUFoTI5i8tdi^W6a9.}ÝefyAej=Q+Fa-1xaLpts:t2q3Iw9H&+Ncπ ^:Hh2DR|QYx{B-V犳"ušK']].U09_ZI1ERk%O_E]YR?W8vGSW7ez@]R6rHݝbݝjjk,w0}n D_t?d`TEnqW䛪pQMg `6 [bgHg9ժ|v/(qb?-≳G]qyu(Ig"}zݟ]UOXzeAij+\f%Ģ7`=RpyFW)wV;AӶfku(8W-G)}9ԈˎgiB))Xbuh2Gluh`G}!ɊW(DUѧ n~9>ۓK6-fah^6Uh O{ן_PKo*h=PK~Dmeta.xmlAO0puP+ j'$*YǮlt&NRRH=y$;*z+ERW+ ߢ"7ۭM |uRZjfJ'e y$asrPRY{FH۶qH\.INGnX( (ILlzvs4!tK)p ]cKXKh/Q4^vEH-,Jol*FGOee8uqg EJ Ni >ryAdjf!.hl)֩ \|?DK/7 07rPټS"2@ ׸ eH8ӫ{:fzoM?dG#Ӧ?FɹPK+gPK~D'Configurations2/accelerator/current.xmlPKPK~DConfigurations2/toolpanel/PK~DConfigurations2/statusbar/PK~DConfigurations2/progressbar/PK~DConfigurations2/toolbar/PK~DConfigurations2/images/Bitmaps/PK~DConfigurations2/popupmenu/PK~DConfigurations2/floater/PK~DConfigurations2/menubar/PK~DMETA-INF/manifest.xmlSn +" 5ҾF ¦j~I6L֛{ٝNElī|þM춫M0: ַi#JF 9hfcl-w*ݳخIs?_&tWc#tuKF:a+ܧt`r8yR| e~ 5R!`ކ0f3K||: settings.xmlPK~D\.X content.xmlPK~Do*h= | styles.xmlPK~D+gmeta.xmlPK~D'sConfigurations2/accelerator/current.xmlPK~DConfigurations2/toolpanel/PK~DConfigurations2/statusbar/PK~D:Configurations2/progressbar/PK~DtConfigurations2/toolbar/PK~DConfigurations2/images/Bitmaps/PK~DConfigurations2/popupmenu/PK~DConfigurations2/floater/PK~DUConfigurations2/menubar/PK~Dw[}META-INF/manifest.xmlPK6beangulp-0.2.0/beangulp/file_type_testdata/example.ofx000066400000000000000000000114601474340320100231250ustar00rootroot00000000000000OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:NONE ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE 0INFOLogin successful20131217204544.559[-7:MST]ENGOFCT3011FMPWeb2013121720454400INFOUSD092243467384967falsedownloadSince090341355486768trueBiHJPMCPVMUZESUMTMIASKPSHBZOJZQMZ20131213050000.000[-7:MST]20131217050000.000[-7:MST]DEBIT20131206000000.000[-7:MST]20131206000000.000[-7:MST]-75132124581254980455140941621247980353Cvzndybfhlgsy Kbptkt010-743-249287278814438304-062-9392DEBIT20131208000000.000[-7:MST]20131207000000.000[-7:MST]-29.5139251640671720832411944529384600439YJTEJSYC JXJ 38137 80223112202841814901332133213240DEBIT20131208000000.000[-7:MST]20131208000000.000[-7:MST]-96.73518223640481029842349922421383839452TEMSRB TQBHHWZO CZYKCGDX.LAR/CDBVR5D49Q7S3 IWOUXSFCCIZDEBIT20131209000000.000[-7:MST]20131208000000.000[-7:MST]-45.49410313240598642566201153532386740368JWNNJ VPVHHV - HWKZIGH QXWR 35905 UJGZDQD IUTFLDEBIT20131209000000.000[-7:MST]20131208000000.000[-7:MST]-01.7118954331459601590112944250496740196ZPIKRWGV EBQVUE 4521XJT AYDM 227092130 2924489277DEBIT20131209000000.000[-7:MST]20131208000000.000[-7:MST]-30.9118335238578609388542324610398801568SBMUZYXG XRB 98038 03324302420853700608200014392232DEBIT20131209000000.000[-7:MST]20131206000000.000[-7:MST]-39.72141044448255701269230245232285603469SITEH NIHOX HTYZBAWP392-139-73484165624260 423-151-8227DEBIT20131209000000.000[-7:MST]20131208000000.000[-7:MST]-22.09111921351569432591101153620388713392HGJQJEOB PCQ 08418 1KVO VVDJ 84408170244 1144012409DEBIT20131211000000.000[-7:MST]20131210000000.000[-7:MST]-22.14548935642111458816328141439181292814LILFN DVRIFI - LJBPFDT HYSF 32793 IBTSNAF UDOMKDEBIT20131211000000.000[-7:MST]20131210000000.000[-7:MST]-12.67330054241010450007342912468199362629PEHKXNPZ PNW 91458 21015119128823891919293222482430DEBIT20131214000000.000[-7:MST]20131212000000.000[-7:MST]-31402252668937162476448222302958184678YHVWV NNPYW HRQZDUOJ201-251-53365093823538 334-432-6338DEBIT20131214000000.000[-7:MST]20131213000000.000[-7:MST]-64.25542124402167304547222333308839462735MRKW'G #814 HYLDQF OVNN PSCS 89318083925 WOQO'GDEBIT20131216000000.000[-7:MST]20131215000000.000[-7:MST]-42.01210943512087240724501152389299358014ICPBVFY #2321 929883SXU QRKN 12054513980 3062749128DEBIT20131216000000.000[-7:MST]20131215000000.000[-7:MST]-5310215309277250199448945618978219897LON BXBZA 409 QEPNUAMCO WDOD 31345520940 4169979285DEBIT20131216000000.000[-7:MST]20131215000000.000[-7:MST]-0.69430314310166238818109924680968112897CFXUF LGTXVL - VXCJCNI EBUH 308496 NSIWFWL RHSFPDEBIT20131216000000.000[-7:MST]20131214000000.000[-7:MST]-34.43330252321978428019511914698876210997DGCV GXD MJKRYD JLC IYLM 67935504527 134-064-6852DEBIT20131215000000.000[-7:MST]20131214000000.000[-7:MST]-24149941491182603315429354298891572407UPTMFSAD DSD 12354 14003323410963402919381906089541-3609.0720131217050000.000[-7:MST]falsefalsefalse beangulp-0.2.0/beangulp/file_type_testdata/example.org000066400000000000000000000005611474340320100231200ustar00rootroot00000000000000-*- mode: org -*- beancount: TODO * Dependency Checks - Create a csv_converter module, parallel to pdf_converter. * Misc. - Complete import example file. - Add tests for import. * Import - (cleanup) Convert all the importers to use the DateIntervalTicker. ** Filing - Bla bla ** Payee Normalization - Bla bla ** Auto-Categorization - Bla bla beangulp-0.2.0/beangulp/file_type_testdata/example.pdf000066400000000000000000002707541474340320100231170ustar00rootroot00000000000000%PDF-1.4 % 2 0 obj <> endobj 3 0 obj <> /XObject <> /Font <> >> /MediaBox [0 0 612 792] /Annots [<> >> <> >> <> >> <> >> <> >> <> >> <> >> <> >> <> >> <> >> <> >> <> >>] /Contents 15 0 R >> endobj 15 0 obj <> stream xYێ6 }sxE( b=h(f[$d)f4iǢx9( p=_>^c@s'ϘN~:/ M0՟_.|p{pӇJ޷~Myz| _|9?MlXp CIeЉR')L #FB}<E0"a$p0j^Ãr̦#Ӕh}.HiUD ,AE$ jJ&9( ,9ArN$ HdP+Y]pHkQX֖!|n쌪$Ww7;#~,O3|pY\1kjeݛЧ;]֭+? {&XL o w_m14~~|Z6:lRZb^ʍ@wk@F@.EqPclc$=d('˪&J͵)cA#]5X_dDdή_=/ȍ͖#7PŚ7EK]GnmC39k-x#u@ּWq'{Q4@cePM%EkQV$k>SV0VOŊdt7:44G"Y4JfkA7k`173)ьU*RL*) ,cXU*Crߥ3ebp7G^pqg9Ǐ y\v}F0Rb^t!^-S(ZZlX*rV VGࡔόr/tRI 7WǞ]R s7%RC7O,oҵK7O?,Zdހ 3,-9b |w\_=e_Enz+~ endstream endobj 4 0 obj <> endobj 5 0 obj <> endobj 6 0 obj <> endobj 7 0 obj <> endobj 8 0 obj <> stream xwwTi$!$0810`PF#au8Kb;N3_8u ސ>綹{4|뻴d;:~@@HHHHH軨)pAd*?2j&ԓSF/x⼓O۩gsۧۧ}gBBBBBR^d,?2Q\)<)W l1 P44g&3%'5m:m*24<:@HHHHȥO!>f("ӄ&م%, ':6K(i<191FHHHHY" #W,?9-ya%X MNI0t*P@ 5e5EHHH(մPū%7p(4&JNt $$y<4*"$z%XcoOG_?Eo%%g,H_L M]S^I Pj NZ'q 1?>_JiId:ݩ(AL!JW!N֚7M(PB-)BB)$=J| Eɇ%ph1D% .6yB 4=-Z'(8 OL"Y `P*h1рvY?H}#4!QɓEg(H@ L$Z&Zd5f6BB 8:b`rL OzdϜ,Pa$r!@Ѡ&g_%K*Id61׌l ٶ h"3eP4:;  Ptlޟ&41@yCTM ?AwT=p=]·}%ͥ)Y"3"͕$z-rJ^o˓'L"B2?GyP(τ%q\$4<]&tk(B?~ ??(P%={Ý{}# Ms&L1`%/22=U!G(:Մ/$P$߇ |<JAI߾{7lLWz0z=;z='z#M(2_ _$TP2?z{-qz6H4Ztf_>`He@1>s| $xt&DI«GƒG 2(?ojtXmu~﹫i.S3E&K02a{ A`HRhh чj|p$u=})>mؽs[{}"AIjKk\ʥ蟃ɾu;-93 #b^4х'D!|Kc#9[!_PcuY~{6Pvຫ%L;5@spe'5F(*}v)}gOI?Dr_I [ SPPPb1r߼7=q5v;q-@%38pDFIBh2x AG?v[&1ÿA_&ފ]u}{*x/3_О&6X(h `-4? {tXZIR|`%u|)޻%[ b"s$#Gǝ&4E┋)C),yoM9_;›1 AjT3Ut`?vUa{+{dž-L4̂n) f)!<>L"FȆ!C F2_d?f5xomj$?D+Rv̑2jh0&ڵuW4 !J(͎5i'zQh'_䠑,9Z_U bv> kg_#4(D !H^Dٻɢ M,`i 2,UcHvd~mԹ.Q`GCGH˚uݶFmT QtuL)d`쥑Hlݣu[~zf=P=1q1k.b|ʉɏP>>\|&AV߇@q| 7SߓXHm_% ԥ>𮒾]%=;6'L2%/eF@sW Y`I]\8JV>){?x[jS,GSM^DFG .Sl-i얉I %mc7( Ě#~DLjLJ--M U]З{d:S;[+zN56@&=ML $E,6I-ᬭ&?ִI3б $+ď20|3N96w*MV$5@s58Jo|#ֈ 4~/@,4|E(E6Z,:@)(y}%4Y32Я;Okper P]T8+M%*5֊&R&au8׃+ JE}jhm{-~F((1I<9^ϟ 0tC KkgQy`o(B|nȵPjow4DJ2%mM=ݣAJ?iRb[c6 ־`M QBCgk%V/h(ޱx=o;:m&uVtխr+%߈$My]/iJ<^)ǥF_n_т)\4+,D<ӄVٻP,١[ ]閘]@\X= 1pd3i46F> wsmr#߾6moSn<[$JV{G_@Ȉ~_/a$#z?EϽkO30ڦ#xbqO.wAv{ pS`'ٌhW5^ =nv2:N("IPipOyPBBm)YbP%:@lBxy(2/8(@Y/Pre4(1 9.GH4y/.J柘\vg@IN7/DG&P6Njg݌;ǥ滶߱NA co (Uib %cݩl$p/8la:tI*}}nWb`2'dа[ D~?8QЙ^yHP^^~K w4X%M^KM@cd%}ڦ&Vr\dpO Lh5_紏)fl_ӵXIjAW֭&tUE+x(m4IJ=ܻIcmTNc_'nUڞ,AS Ki.߅K/ PQBire%b@I .iv1p!.OͣdZ7>z Oq5MT\c}hsK%0#aR[$W`U+b}jI24z)7&T8ֱ}ҤQrM.5&,y1 tϿ=ԟ[w1.3'M6z2ɇX!6}b2n'wqE{.L#Jj;WJTriF\})z8>.\pG0'=vȫll8 6J&!we\D'D\8rQRt^M,rmy;&v?mWP* WLRtoW 6HwWPcSÞ=ah k|tרY;Ihk/P]ŗ_~ޗ}Q/M9ͥ?gd/89&m3'$|CM$x҄FWW b4HU֚VVC,ViG'#ĬJ(&&@1ɊĴIGJzZOTI%kxt@vDI*Dhredt>/3MTy'&qC%-ɮrФ{;Kwow}}#4]8¡.Y'%|Є6UFwkTwռFwV@[卢WFEMtC L^3 iK1dCxV Pi̷w<ck/pj $@Ɵ%h f>?m[=@cglLAh ڃX&@\ Cпmo& Y$YUCj5j0B"5VmH%NSVm0Sv;nn&8+eBo׮غ6֨Ȇ{ɾp:* D IsJl^aGź J (zy‰sgfvJQR; JiOqM$UrD5`CA-P".)z`d Z%1yսu1(%`D'J+ /|!'oyrϴ5ZiMUw+B%Φw6/z56&M>G6h `dO14a3J><2 LЄ"D:Ϋ+۶U׮A>(G,0!cɠΡ2b/UBwXm5GY@moC BL9J:*I B! +WJli<)ڑ&W\n?dsKsd.%4YX4sM"#L8;ck`QR :j$ rZT-an. B_h`81o41I성JiBPB}gӛ*pLs]y[Zu7D*znNq/X<1'Y8 Lir:4iD:W P+M!Exo]sozS ^k()([ԭb: JjpqOժΪU>H?At X# ѧĪ &JNx٭pQ7'hS*U,܀;o^~W6Ĕ&@PbjS7~,֘% M>gO2V"_2H&pƒ(laM`&!I$y_J^opd <(eU` A9)D8E@Iv$e::j1qV3xљz7=88Jm0l1&& 2ovW.7Ğ&Y9\Pr9 OL}4IdV FJFȡsZ4qH|(Z3BtC+ f %a߃BMcVk%m*ic㵑kۡ=M"M@]qfI` %w6ygMX~l򜧆&[|f,$i2NgqФMI+?M" b3M? #[I0BZY%8k]M$5q?"4]ɮĖ&4'GضC͖ԥ E7`4mE@=U4J&X2d.L# ZEVs&m.‡\7A .'9< ؑ#(Yc-pomqWuqGU 59%P&Ӕ&)cLW@#*V -ky`B.D &42ҁ׉XDk 9[2IvJr" Z gLڦ]!%43`bsP.Ɣ#}1Hq[)]]+J} z||z ֮IذKª;qכ7yWh˒&h~\S(9όB7}\I MLڵbr侚b`Ĩ1x`j,9R+=J8r#J&1M+: NTO(5\ĺn2 4QHV!Px[4ѽlN#e[-iBL J6}$}Wu9hEmߟ0)zgQI@I^d  7ɩ&P!G;f~,GR01*:A', $]hܙh HPbbGI@? fu>O^6!0]aJgg (z.pD=kf @)>EjLF-ށ  MYL"9U`lcI|vSKQ(&(A4Ij[SL6NwK;En gXӉ^x}m "*#}j6ݙxNYohx /D&5(~#})O]jMܬP%MYgZ!ԥK\TL%M»π ()YoO D$L)h8'-VLX"#JUX;#U4Jfvy ):/YlQ4i 63<1kr@Mv s\ Ĝ&/@Ydg\&D: B:L0~7?tYd"Km*<)` "<4^[ X h}%OtfWTҽZƲ]!JC1@Q 6@YX^4?Jf#. K^ӄ)M\X: Ü#(IFׅAt&-9k\XC(h=.AkxIfr^]6qKD YvO@Ix ѦCIJ渨J9"w"@A.D?qNRFh>Lӆ7/KP2430 .(>^ƉK*^rd)%E7KyIe@9/grɟS1EE3]ѠdP:% (,|Wf&^8$ny C_q7%{yJh RH%eT" (&ta!t͉(Z6J)-P|y_i Y9!Ii`>m'MC$&$QB-mB`rsGĘ)1_/o~SR@FMTR$RC27/M@N,/X*EJ߸]Qk=7Nݮ“>'H\%%*K*N hb+@xZ~H f|Ai]!H!;n򷒢mR p~ ۱#̆&P(SƂM),5JI f0[ϐ[8%Ծ_]9<-$,(KKᶍ4qD Z%XK%{Byg|(=Yaw7/KfNbw^iu̒¯J i Ri /}I@ j0AD]9t< {x1ME͊5J7K1I H _=DdP!hE^c<dʘ MƤhq㏂g4 g]˂ԏ$qI@nyަH!IRJ#u%R5(JcEp#6KKLil5 U*d7 ?b%Hw~C#GL}l,DXD C((”T)Ht‡3VhBRmt?78LԔ C'.[qI`8QeڡU4}wN a))GӐ9B2n/mUkp/Sn&ZrZsDJZ&2[};pWۑ 5';/Bٴs2pbG-;f5DZ qE Hiw 7IXG S`# =,cT4 ~f,3$N4 E(tU.N-ah LXLV\ҥ#fdb*AӰ&%P'Xi:<J<lD|8(jyD%Կڅ(%K_(ٸ.AL!b1Jn|~{᣸iT |'M.c΂!!0F3M9otEI!ė.<ӥ&#>Ai-[:|p1 PV!I{Y~W,%ο". (; J_n'?`+M;8 7u!D88`}z񒿭{=(Q)4ő pN)BWꇃIx*^OE$D+z A(Jn3tv׉nYIw)VŠ"ʤiHn݂ksNLIsǺ VX;TK Mds;Ew NqO2J߰z]!+D\r%@B=J$g@ɝͅtSeE=wXL8f&`J ^| Jxg)"r.PVRZwe @aG4;cB-QbgBWr$BQNa"KEno اֿ%) 3;?1#pgR>v@I&4|xۭ#7K I쥖?K8GΦ%G)H"khm!%+J4#뙗7(錏_ "*J^HNp L(kV)VI#IIL('Li}pDEIF+ éC%hҡe7*[L\O 5KGHsȢhk"$Tdg%ԙ͓4u DI%J~}X"~DL./"q)q2ak'CҚ2X~LFZi(d @$beʝ GAĖ#*˖~Ź7jDlY>4D =%مZD Dʊɖi0Ԋ4!Up<&$ ,@ʎ!`3zpʊKSG'om<5ʫf:YVR'})S|)Na6F9 %ek(9 -> B BMG!Jֽ~kR+B"H!]Sf\/-Jʦ;Ǧd.>BĒ#d q2 PFR9 DЈBiIRZEZ+lu6ٳ[` #zyr֋Wݙx9 6W+$$$]@ i6fwV89j·3D4Ǟ29P/!u }$k03$҄w(,y5cc:^HHHHSAjVȿkİy~L⅄Gbl( Jq֖+V+bw2Vŏ& b LE%Z~n h"$$$p:V@y3Ɇ'ɾpU PpBidCrA!!!!lF4Ɖ(hXnq$څ^PYbf0a$4qykQA!!!! ٝ_źG{;BBBBi!:/ TI\dd_& ƳYiM% ۥ&7Ȗɾp;I.þ_&[1˱U4ʈA!!!!/ +mɾ^!!!4V&/Ѥa,]jN S}BBBBi, GOFXP! yP&FŏpDI㨠JҌ& !{# #ɾX!!!4o i Ms֏%bXGgP}L4v&xL($$$^{ʈQM& iqfpHkIF}zDsR #AG$MDKHHHț$p[4q$n(+*x†#ԏ@`"\BBBB%.7dIgpƱ YHHH(-D;[cקh88aьc y)R 8zSPH"0i,$$$䇤GnR@O#6i}a8T pQk4P0U?JGh qsQ/VHHHQ~}( (dGd #⻘%$$$ YLC4%safFQb03( `c)M#$c GF^}T  /:xB ?xJ7 z:J9JPsaBBBBARpD@q?yҔPP2"a{Qyh*[>1Kt8B|@6$QQRO.CQyН(,&>QBb‘zBVr|.B<'`ASO C7#ٞbHM05Ba8AD2uQ+J?_#&d):yړp?lS8HbPrbV'v|ݒjź!!!$J. RH!ANƥLJ2,hBBBBB =7P8)u8GDHHH(DW`%S%֒P*P d²eg(ʜa!!!tP̚AIVq$MlDO!enX7|P`$BvV#uv!!!:ELOHPe[7YtͻJz2SKl"$$$4 Q>G֑d}"JFٚE  }$zG ɾ !!!!!!Tŏ^ endstream endobj 16 0 obj <> stream x]*KH @ ,,!%X%l )!%x?uU9!^ο]nѱbsZyh| LZ[gcC K(>d:Zgc rxbw|7Դ$/.80e=ůq[f#b gM"6ØU.^jL tuqup<(ҋ:I~"ql # -D/k#>t@s˘T.rQ֖Q0nYhlu ÙtC[˨e^022Ĭ'F)ZZ[c  -@\kb7RL5w(E;0nhfЮ|,F)ZYƁҎQF&+.XQF׮nhcl,YdX[m,[veP]FG1ZXƞv$ePW 2xO X22xV\G[=+c losL6##Gbs˸0BU# [[ưՑ+?3e(.a1p\{!UA)<78pefc.y1 &*Q͠h|1(?a?N0ܪ+ )6b#)[l0#ab><<2z⥱@-H[ω6dM\8Ɠ4/X-&,,iDp~ O# ?J?T<٧ 6UWeU 8Eɣ<@f )yCUepނk}L9Ѝb*B'ǾooiCg| &]t6cps_R1-xT]; _+\0dN4:߄$QzRC[Ued;r}iU%4 p ?aW$X[2hY2]ʪՈC]g~ |agX^?Tfp!8_~UuzO'tN "k8SɾA 2Tc/Ue .#pt.2y[Lm%|@ eUe׵cDy>2&I2p뗭+0 3فlCJAx_CrW-fp{tC uP: $}{C.L;6 |*489H4[?Id-v WnNU[נ@2q5Qo>~ډ:Pq@͝s43(Ced-hTphS$_ N͒8];+:];Kom6IWg ~T`pO$N}JN ^EC}1;Yщ_KH F9LSݕkCj;#$R!R6dϐS|=yxMdPh 4th7'Cȃ WIgΉiYL,X;}x,^t&b "l8ɱCW[ zD. K@ʶ J$UDb#gk9݌db &"xK.~FanhuFd>> -d 4Q R;jD >>[lp^wjHH~1"DìPoh%?!D>Czv9ׄZTwYZm 4cmBEؿsh`H3W&OpsZw:̽3kU~/ Fxl֟Uo5:gvEBe8!_hfk[Ay>ҿ[`S2Z&tw7xFw"Xw=?,_uSqȜ!zIfNPh.'y(3ۘ!-8mevy#K4e"8fs 9َdÉMΙ.O?`m 'EjS06kwC蕊[ TVP}W >D~]o$lL%a[)0*5Kyr {JÐdNJ4z*4qkw1E^lA ‹eGm|N6Pza'ep69_a} 5yǰd zpw8" 7F , OԏxJEK-I͆7 {uؙ oHJw}¹1א[I 1(g`/0=BnI~ؗ"`#awi@:Ez4n R}0C=욦{2^phӍ`A}އ1-QdzI|i>V} e KͮiEBve(=kB`却};L'ې%^ !pnNf/.xDlC{cEKrLB=9ZmY>޻vE gFq 2O%vKܚ%Q/1%ϐD-Jpi[n ܼOˣZ*[0Ski⍧11"4+8T[}pHS#,'Sw~9ϞrL r&wN]ʦ<;  ܅9/F}y+{߱*Qs6w2Ϙpoxg>@6T|ȏ8q_)ķ yoȀ]VQK\jD:}Jy_=Zkcbnw9?:g˲m NH,GSz 4}c,8r RFa\=aa>?6vގϿm3+]40}c:Pc '2cm9M2۱HwDgc;ZO Px(}=pOlkW_fZ"p7'H&ndck0:{n>> #c(`WcGB\ouBk~ȕ1O 9^j;2rT[Ʒ\;͞!n_se|\(t*ʖtU{jIݟaV {QhMGsk>*[=. 0 Vؖw>e3ؖu}P}Z~_fٵj`U02OS0"o02 0bԹ? k237 ?u<#Q3^MJ l}@w5 ;C 2&WO^!0A>djǨ=w}$ى .|U2SNo_opϛqx|Ѹm;A]sV!R H{ADC;9osݱRmcRsܱ 2y s y{س7-08`GS֓yCkTLutttP endstream endobj 9 0 obj <> >> /BBox [0 0 97 30] /Group <> /Matrix [1 0 0 -1 0 30] /Filter /FlateDecode /Length 201 >> stream xM10 Ews%ClHN.=*0@j;PgV o+Ne13<SC0VQ hugkO8}1J& &po8{W"P%=ogV "Ms6I8Sv*#}P(n)@r[=k{0^DQ endstream endobj 17 0 obj <> endobj 10 0 obj <> >> /BBox [0 0 112 30] /Group <> /Matrix [1 0 0 -1 0 30] /Filter /FlateDecode /Length 199 >> stream xM10 Ews%h$=sJ/vRY'ہIu$Gp8Y lzķx]J2P1bcpS 6٩߼Ch+Yn*eH33`2z5~)2N`JE6QT:JɶljEYEy$0_. Q endstream endobj 11 0 obj <> >> /BBox [0 0 17 23] /Group <> /Matrix [1 0 0 -1 0 23] /Filter /FlateDecode /Length 103 >> stream x5K 0C9E.Px׺g EBDMN%B 2D f֔YT ]:Mw@YF c+h endstream endobj 12 0 obj <> endobj 13 0 obj <> endobj 14 0 obj <> endobj 18 0 obj <> /W [0 [722.168] 17 [250 277.832] 19 28 500 29 [277.832] 42 [722.168] 70 72 443.8477 74 75 500 79 [277.832 777.832] 81 85 500 86 [389.1602 277.832 0 0 722.168]] >> endobj 24 0 obj <> endobj 25 0 obj <> stream x} |SUo֦MҴI6%iҽIw ]ҕ}Z@);(:": P*:PQ\pdFQZQܛe>;{YmQx-\ܺW/?? @ֵW =}ᓎK?U@8J`v3\:6u,&ߧlZWqT,ꯑ~xμ]2W,s *}˗wW}vG٥n{U@yǏ[[v5|o4_pQp[i7Oп=2S#Kk;ފ#90wՋ "hZxV@X* dtb_+ʱA,C>W ֛"#4"X:ha # &Xȏ].ʛpXhs)SXr1=@g;WF\?ۻ91i5b/<؇NmBqz( t "m؞DBH9 pR,Lb<{/Oy RBv6΂(f (7)osT TFY -2>C+a{Tfa6YT-0;WRpw4yyaТ|uQ?]E=x+RB\" )As=<,rgOmW LF=D߁sϥcx+> )PC0 HQjG+,9lQ^ ܋ mù>٩ޢ:JLwqTϙ|Cwg2wEP#$CUKy FuNS{:#|V_AeApdq06ȡGʺ~㽧w}" 4`alB\H ڀ]K} ];6"C.qg*! 9>-~YGzx\HWI8Dq⣸.C1Cl$ai 37jq[?D?o ?_g`a'KQzsfrۓm#W#~t^dtQ|sq" .yv9"OJq9Dbށ8n}UzJ/#_ٚT~|~ YsL"*MY\JcPsn1%e29Y1f8LJ:OMT)PWU5*UsIq.P6@y4 )cޟ>0dR\D6R+Gp8h{7"95䷏sci4f~T:*3 e\NJF;cD.1sZ 9jv(_S^<]kGsacGp8I5Fdب[<!L ? iz#[ԓ`^r{orE!=!P@hpm+;gFA8>!xM3؛1a6 ׁp9wf=__ g_B@ߋf(g~p?A-@<:3䁱7^:aRMve!BX_, aCjB0 NA9BT:2iޞ@m& }p?<C0lGQx  OSg4<svnx^%s/k:7a?mx»wxއ0|G#B1|s/+pN7-|)@"5"zK HRDEH#E "IjRKH=NFd7f'n>|~_Ǐ~-__o;]nU/D^QP jA#h%D z!Fa0N/L& a 4 ӄ &a0C $"&ICKW*l,v /  )|&~}^Ow؟?oѾ on?j~;)^JTQ&5IҳsNyiOzSzKzOP\꓾KYNmrle!;e˵#ޡu$8#ӑ9UXirڜvgssڷ^RXfz|q/5@z DE4$0=6;l8!TC"$24d&zMLwQcB=9OXazn X8O@=v>1=ϡ7[mE=6_7 n:.)[xyapHx_XDT\Ǜ?dbo?m}k߱؏m?iV*0i\XgH^^ސKHHGzo$(9N'ǥ?3ϩ-L 1QSUN(=QN^Qʇ1ck%|˃>d)oa_; {z`JJQL٨r'T+W (ށeN6|=?gk畇yFRJy9X>p?)`(nrꔓf8TBFsVt+⨻pϕ亝q\bFpSjXIff*h ڂ|P ;[`Kb/>w N~ 7GBnpIG0,!Kl אaKI)_%WACDd *b/z{ 9p`KqGUq:L%+NŪ) Yq)O`24H޶-z@z;gSo};{+znܳwPս=W--[m#$z{szG᣿~#wxdˑ?yȬ#YsVKLʏp+>J9#)<|뎤#aߡ/oq@*CQ2#BBKG!d56B/B^xfᕏZJDX_ydxHgiʔCOrxh?%{N-!~3~Mg{7JxEWFy?=dS9c!?}ؾ ΢1qřG;Mϧ? v>Cqٙ _F4 ɑ9l,9Cٱ:*Cy$D4o!?va"FM+#l*MȈHJD.҅.uoXc Qu&wz''}%zN,EGwwn䀧w"{XGP?;e>8qX!}?3w,f[~| X$TBp{Y{g69ieR(2w8a,(гPz\;P?*N,a;o?1$ Lt&shqY$'xshGGJx@v~)x\Æв+T씱>LM>{^tVO'1eP-׹7j]r7`|7Ժ>FfFv8p\gm;L~y8?.' jPjWYu6Wm]gqjƜNR35\՝Yt]'49.r]fͤT |sc'7a괸j;-+{u}=aLZjK#'8-i 1xUGkfΕ;jW3Q0չk;a\CW?rwX/@ȩʩxzy㷡bA'Go@OdoF_#;Fg%SInrS7n-7oހu'JU' b\5)o&="Uœ+!C ՐQ|2Xo8lIWFD*zPC.kUЃ&M j(骸3ߐ$ r1ʐH*f0VuM`wq;2dt%us)dn5'r5>>"[ӥW /NvyIOg>Kg0%E{tOkTuCʧkq%dJ%]3)Rnv44޻XZix-CüO|WHU iH-K*ûl{y,ܞ9)Orğ#M.Q}jڭSfIch5:F[[9ϢaYeH%ЧhG\(戆CբܐZi0מ13:'V [R;)4,~魫AQԈeGVml5kƆvh1x; (ybȠn׆yqVK^nonen`]\XǺsixs^Nc|tl6QAv(]`m{9;l+0Kƌ%Va,hǏbc!ft`3!C1y`)T7 `#BX o=܄YvxZy1ꆓ`؀\R` P6+bv{|jl0Cus1*k1= IRl!柵,Wp5=oAy9-ptD"Ҧ,n"{de;fƒ0w] D2w^W~(5?I //WM'd2Yz8F !χ%!x /[(y-qqUj8oxqrU٩'"bn_<{)\rޛKzgyN> q 5~*pX! >Q(P^WsoK¹~%1goAY_Oa%N߲aNRM. +ɽdG8If}ۼO<~:+7p"./nS}^l .^|VܦÞdILbUrr5w͸A! ~{95g\s#;lk:uW|3~;AC;&,. I UUZObcCЅ2~43VprPg{'|F1 !.{qOvrF CxqbfTbi2hK~SZ{ *GOQaqA~BY"gas a+!~+XE4yh]D1+\YFLkGf2#E^o Hl~9OogB fQYt ށ^H,4Mv1fS[g4OȺunNMNAe_%fD{nGN܂GosEDTTbL8|hTE7G2࿱^I5ASXƹ*lI(0\Tsgh().VwTlRp;^Z,x+MTqjM$&}t]P{;O> #*ף(?8`?q31,',e&h4Fmsn{2:Zh>K.AO ;4c.4YJG;ENde7o4'XEXq9U"w7LXطb561,EFg vUF_~!{9:[o6y0-ܭ'cO1Veu,A.m.Dxl5јhQeV]#ƛͯ3> 莧c8yOVjCݯ3VTj*Uëj`{}`иz(K6>X3"*Y_+;/N(/.*Lcǎ[?[ H&PEw~]Iz}SIER`]ڄ;o?t\zX.ቨW Z!KLz=O : J Y3uѺnbk>O8"pB7)6x3&6O@TV׿-l>uٚ#d)S ~״XR,nCy$TuB,hu"\i<=Vd Tkm3ΌGNŻE&$YpA*"+&@Fa!jW\L'tDٳ{2H\᫭93~ݭG Cd1ea;<&} Ɏ.!gGHhIA&rZ9UxFC;f /&[r,t3w ku9h[l}lG[oS<~>;wEQuQSjVjWVƯLJXjb,8N/ Pn#qUQ.%Y9 V6wjH/ C}xawoU&äĔb@:=̼]ji%L((>mPV>sŋz얶񹃓K+W}WDJ6юR_(!~}U6xI5'zd~;%]o8{'7kF/0͋TB2z *S1'c5if{Mcڏc5lXyx*O*9yvJf=*NUJÛ: Tb=}aQd P6vae$y&,I`D{ 4 zK0F*MP=DrrEA[:XyUrWG>-cQIyKr-/>ɁWKwJROhkP`-jfK(%[Mi\~ޠXUqaY6!L< UZ,S3)ä2c D5WQ3u'a Ḿ?Q,(sScb!!aSMm }>2w @5++ ,i9ڰ-,81ʆ)pr Gm)lew1: E2- !Xɡ}5UdV>RM ΄Yh/-;O~̤3>)*eF} B.-xvε-:.k^4|&ɤ ϑ_̥>sŻ$9Vck6twc͌/|q /CSh-6 gsU&K'_ >9Vl]Y^uHKڐpٌ*'c!"^Jv]˛Q?ę\e^E FNͯ`ͅCw*T[cWn,[7lYm_=x>C% C#fr\xz~9Hȩ )08*9z(AK.%hK33C~ yB1_QqI~dOMQGڝjlz09Y[:kubJH-60~j:qB3pc3Ɩq!$&9c'4ex⣜`H#NN]%CdɣeW,-[3 :*|ٽlx3 ?~-\%/+'I7.\qn6<^]mo> ˃2hP{bǭroqߟ֓v"Msi#.85}rܚ.lgIޜɬk(ȑ:'i*\$57;# ?FE9@=-km j xꣷSRѯwLlt^ͧNŇy@xB,]Ԕ/!DzSH8DR<<~ޟn2wATiG/~OW/Դ6v8{T;$RU}1;Zs5P.Цq| ~Gđ.C_onׯZ.Z%OѨt@o̴&'՟I6bmXmYtǥ ToRQ6MqzMU>HZ|7@Mcz($}MXz 0b[i5>6Н c^b`>g# Fz@P лNKrIQ%U鹹\%~ȩ /]iEAeUKfgŲq~{ۍe66&C/X9f 麨?[ pХ/5@2D،y?Sk$IN̕ڝ*C7T])kT'5'Yl3}^_j`>$ *ج*ʌ,Al%51n1ؖT2]8>iK(\Z,{TZӝ`^MU.\:$A0hx_i|iRcSUkb4guf8c^k֦qi|P•%*r|IIE6sbDc4JF5\g&:WcJKkRRIvpk*{gEOYM@cE!z{mZђZ4V2Q#٣`R(r" -II+*nj+QB9#gK Y$Z룞N&)݂J}S\:V}X+-唖U kkt,\u<|\ʓG_tVj,GFɓh-m_0ֶV~rŭX%Z,׼%ּ!r/ϗW2=jrXM},/nӾ9nm.nnih^|ࢁLW͛G]0gq¶r}kRdj У@ǂr9T뒥2ξhY qu"\-[Sӊ,iS_>u>[q7tm »Zܼt~5VgNk_ky~K%(> stream x]n0 yCEXJGc{HLi(~fA?۟?ˎͩ1Mmy: 6,No;ˢU.1ʒ}-|W;:N }ە۫0\ sg_x}ּ~U|,x4rR0NX)+^N*FչP?j҉FJjCtȑ(Cʨ(szFq4KH 73R&(x$)2Yuۮ:. n  endstream endobj 20 0 obj <> /W [0 [750 0 0 277.832] 9 [666.9922] 14 [583.9844 0 0 277.832] 36 37 666.9922 42 [777.832 0 277.832 0 0 0 833.0078] 49 54 666.9922 55 [610.8398] 60 [666.9922] 68 69 556.1523 70 [500 556.1523 556.1523 277.832 556.1523 556.1523 222.168 0 0 222.168 833.0078] 81 84 556.1523 85 [333.0078 500 277.832 556.1523] 89 92 500] >> endobj 26 0 obj <> endobj 27 0 obj <> stream x|TE?|fn߻MZ PB B%H 4RdR,`l " !3wwC|7|zg̙s̽Y AڄI%'(z=nؒx *; 3$ kƌ2~ܻ'x7@GZ%#F ƲDcc.1/h~l؄c߿gy*@MCM"b!cOx#`_]~#e`u$$1?a-U- cyP< Uj]Y斛Y^15e)x Y MނfdkK!޳bi""mKރy=m#\x٘1>Žh@7A& =HSqkeGٺ>yHWd2磌/LtҬZcMR +3}Nj:^{NY)Ё{@4L`Y7=}s + q2J؇܈<#O'i,FAa#}Lcv?=!lDQQ8i8lMHI棘 QHry4)EDۆϻ>`05|)RzH0.Cm0ZrtՕ%k]tWHE\G8uo(G(60ш9ynAx |֑uXW.Rݶ~S|~d:H瘞Kk/P> :֋Ns޵z统y+m=19B^q?PcS j=MRO:>;*dX>{Q7Wiݎ[`P:PD~bG'\1l" ٢E0;Ȇ~zTFY։!v es^ &9qh̿eޣcm'̟xdG"L[$v`,\ivc}Wo=Yl0b`P fC 4s߅ǼX聞fPb,(K1h̆gH/sqC ,C,_ Yǿ}K~t1ƗC> σq mE 0k'3Qu9žǿ>Nv_Ϣ)C{E~ . YU _C8ߔw1?Wĭd}BLs\ 4Cď@z}g`k&)2k3Wn'7?y;??:"u"D i'fۺi>]Weꬍk=ā" Sdy{a&@[QFND0iÑ>P'aq4G_iuezyWpb~( @{^ҷ.?bYHW 4_D F?}:Jg3/C7wuufPz{sk}i8յѪЧ42?G/o(|g"]z}/ݨ[5X "t4A#{Pýы 7#zE凶n/ߐx4|6ͧ[cbt/ߥ‹ um-]N}v53H._'?.m_%x])Au߂-}_#rk@ ]-6J[iɷ ] ߉@Y} Pt5Yη*?q X/[лs]e"t=Rf{Hl VsoD)LDlƸP] ;Gj@j@V%>餟-Oٗ_?ӫ?ɨ?G6;x0;lD+WC{0ہ՟ 9UxS!{;KfS^;Ǩ} 4@"<_ ܝ {Ϸ܆ JyVC!O?oZ=319 xNL֟Uf쏝eFԜiz\'7:yhKPs̻6hC=+}ީ}=jl~sЃ{ }3ٗuk Rw7p?󼇝Az}:9LkyXB89{ԟϱav@WkeJ7 ,yn&Bdca;س#`mh>PC?W m7IY`8X?=  ă:\Xull\+:I.WύGe(N8~0;1| P!@/>q3{9JW@x20 }NX G{˜ t^,CTҧ0ck/gWX>/xi5v'e4.0QG]x#m[/w Lw/W˯ L/w| L~t LZp[}~? z?q ? ^x.ҟzFAQ4|hO!}=moxmz~fokūCzfib?.=<桺oK^5եu_Y3ΚrBk'sj7nOgn>xnSOT~,;ٻ9wG}- 4Nx!dk]h2`gx=E/1pɺ~c9@+XЩl?  |4}|-{DM}{;5{mxQ4 b#O1Z`*q%(O%5u$po{߿E0:`x#RO{;}vW+a=Y+-hw:O߂aw|Z/~Z[vRȣh/N/LG_mk<^ 02dK^<Γ~_>㨄pRΏ=H.b> y705Cߊs}Yvp&Kwn}ޯa wr<@ ѨOUwz]'q\=H;xb/ ccDx !C"CLzz%M,xTi7Xۻ`(C~r:05krwJz\7,bl׳g +.#\<0 VA~,?㜚Mqx̏:@ni0~}/K0RR*N,:<+dC;ڱMd톱wkG|g> qt#GGyo=g~3q!:xy>>(=j%qlUz腺QN-/UMc՚g;{_K{g_Zϟ1|ywϋnS|Rݍqxg.w{^|<+{ qx s*)3QvN+_Q7k砟9raADxݿVs&ЇZSu}ꛡLЃAoyt9.%bT$Ni <u٣83}ףzt=e|={<6U6I]sH ?5n׷Xw u6z ڒ@&lf_y9<'uy[ߞsnVi-:N3>~~ng q5o?$tAϷ5ZEn8Bi$冀}jaozzzZmڜF4[[K[-666m]Cv}}X+FZccc;-q8atXA0G#đ(urs,tqqg^vzi?~?zzi\çNNg?w*T֩SNeJ;x*T䩠S'/䅓XN=觝|K'=d'cOOƜ_qͳN1,tXo\IJ+q?M_>2(7W949(pu Kr,mD4I27r.( !,Echb iR'W=֝`fݭZ5քjB,uğ p̂@9,Ux,P, OТ8al_*\u އ!!( C8 ^#8 5?Ó)| .|#a410xл Sa L{xaxa-LǠf&ȿ%p,'+J,TA5Ddsy&kZQ#/uH^"/Wz*@6Md3yl!nl#8N$뤜&ďX? 5 $A IM}-6 %a$Dw~IH4!pn9XI,yOC!9L8OIc#1| $4 $ r\, EbqT|J|Z|F\&K\.ċ+ĕJ\-׊ŗėWQhqA(n7[DU&nwcNqX.+7=Ž>-mqx@|O|_<(??Gģ1_W  A???'//œ)+xF<+~-ϋoq(^/?39|INE@)H BP)L "()ZMKj~zNF=^PUS7fnxXe6&DLT%R$% 5D JLi4[#͕I2iPZ$=!-HOJKgҿ ^Ϣ{^Z%Hku ҋ| '4|_I/I/KHW FiYzM"6iCکjFjjUm]Uxա:Jn?$s\9OnO;NrY"wrO[#r_W Ar1&jC5EmiE'XMW3EbyT~J~Z~F^&K^.WɫM7 C.+r\%k:An$T[T[%RT"Lj )TFjfG-ԟ@D`BRI a4FHEi XҘ8A㩃:im@iMVMՓ)+gzEE~6)Mi1M mJA}>B鴔Πәt-s\C|D>*????'//)+|F>+-oO%g|EE**_˿7t/XR%r\- B*k (DBp H(AQ!D QB#XbV`B+J+!J+JD+1U)v%V3Cq* J%QIRBPRFJ4Vҕ Ti4WZ(JK)$(,6J[*yUW$db2TQtRJMPz*JO)0Lfɢ*{J2P+!PD WF(#k(e2FS+$e2ELt1]BKSi ]FEt%}>Gj~zW_U7MBzL/s3ln.7[-Vp%Un#mvr7waz;}}}]~.q_/*^wBZ-jzKRU͈GzV[VSp8 BJZ .VB'S+@>a0J'Lq Cc=.2a}era,/ M[!*7;@8*|ĥ '37\c;pYU]4Hb]hУE@L4.6[5zmNs}w@AŃ!CK 1r1cǍ0q)S=ȣM/Y̝7lEO,^ҧ~fٿX =j^x_Yꆍܦͯmqoݶ}]xcϛ޻-xg{><|1O>=~3˓:} /N'PPPPPPPPPPPP8Cqӻ+=YZfh޴IFzF) $8qv5&:*2"<,4$8(0g6U"Ks@ܸbYq:x`L\+mä;˸mz1۝%]XrX.OIWMIbeAVJC[n}8'VN(𢜸Bav` csb[;oڈV.]!!l5T1&zJA6aq97<ݽGAnN^M qm~zh7۹%H6X`p_r Wl7t77៌C:v;7hW0vn$W6Ƣeesm5= jٵu#,^Le7MHب<+e)ţln%m܈Q85en}[Dkv"rme ȸ9Q[];sRn{ M%5yzH/B=k8KX:@mClؓ8S v)ieCZ`132ҭ+.d鬾[pXleᮧ8Ow oLNjD }awr;)Z7Mi8Mؐ ;vpaTd&xA ÈG'n"+5MY>_NpS˩^C? vΚ~-$K<{/{y'EM7lWERoFrz. 倚,R`t'B=\Q*bs[;x?T]atrw[{2;;i~ee;P< vx]`sC\WkPv!ڱ($o􎂑p!~t4CEWVg++.\gĕoӷ&\X[XAZ⢠vkcտ`6w6ܛ+n[I.={(L ۝ac{D8\hnXZY]\t6) 'U: .-x(ElAECxe6xjYEsQlVK.?..#4V"!!#V#DK؋縸mK3d1zt':HnW]zxhNGObxЄH/e`J& A`'GXa  nDo L_9UǑm&6Vt/yrfm:sr=a:=xlj^Qe~YnΠsWF BFE\FH+Z)+ g#(=W =:W?%_b><3}HNo 4I/oęFz֐ns4sa۲FZ7m5mF7b˟ Q1tC'FՂC' BtG6llkm{?LN? m1Vhb>` R T[Zygk*" 1!ҽ4vPk 8$?eX'klhcg{0նNr.[Qvq>C✵C|h9f9t٭7RNW`mm4#G.ݏ\xz? mIIȱ$ki)CJ{u>FJg,R:&(RCJ] YQJ\;fH!RN&NR Fʩ}[ d{#G(v {z1z 0=)o2}|N;pb^Bұl tew2xoKvw_;-QdKnq!bBؑ~_NL~7 6 ;h%^xۍhk9YM0(u$āLM!Jf DэHӷEj~ۜ jވ:o!b7yzS6~5z0\Ɣ=r¦º^tff}]G[GG%1tvDguM{fG fyJ5euvYӰ ɞ`v61Jo4.FadL*IͤtdR)E6F ˲(2A*κs Q8ثx=l]h"S@.jK@}6^qĀނז!w[wrIn[' 1M4DcI#_(BXȴy9q)^oG*po.t]~9Ur%7g7‚\kr5'KZ旓z9_J/z9 3+69SncxFC/POXsseBm0Y/39V!q82!pH/s(q֋DEa((HЋ]$[d~MzK]&StWt$d{!ئ8.Q^0mD>mBnY|ߐ.qƕ万ضXv0 wm\rno߽I;ښ_VqfMX[Evsݞ՜՜^o t^Uct;U (ő¶! ume{,ÈWeIiòpM,3zke YLk SN a#s<&Le \'rq3y @;W;٭lH4UEړ[D)ҲXx yzi; Jۉ+LɅ;&7EUۻ@_Ʌ8$Ldāه)S!//x&VcI͇1+cS T@8"Bxy'{VԾgQѕ{6 '~a0(Gif?̇0iTXm-Ʋ1a6 ֚ &6"Y  ?CgHV=-^`7V*D~>NA xVT .lK>`%Wmv{`w|{~?_' >.q8G,A]!I]ҧr1;SzW Yn'h[#(σ`(ׅd}b+ڊt+y}i+ '`ߞG 7 ߁KtX(:p +ݪqpgf%n^,%8އԦp%# .T?H;h_Z(*T_O CB&>9!/~Bԙ̨Yv#Ӑ`W(6|B'GMS#&Eҩp- I_ ~-Ό)2(@=v$8DG6#CK2hI*1o]BFҴL+uF(Ĥ42 2$$:bUR g4IiɩsN\ȥia$wEd&-Z$\c"5$"j>qyXzѥkE,׊_T!Ks͍͏ZK欹fF,Yzq)!q ΦM5wkFzHp@5PI5k+"2ތ;\C_!m>loE"ֳ1Ss88.t޵tؼyȐ召wפ'z|GOWo-`#s]?h`Nl~RUS3 IpoZ0N7dn 8o!qٴ4دXrWdGkY,糲 ZVUֵi{S=OIDxY\=Y㰝 2l9\a4 4k5_a~%Ⱦ8-iFpÇVr[(TԚ]g}hvu7IdkBQUЯ4pyRl-,W,,ejY፼7sAxh2ˢ$1,F 63iI ̢҂YgAXK9Fr:l~#@ 0ڠDzvgxn OrB\jw>錑[b$FIG%:]*߉ϐ/׊&#_%˥p˥Kq)|\Qr2Fa:%s-+x(j[ѿ`Rv@~ ɤE#$svΙ J|ٵ__VFe7Ȟڟ,}Lo,hH"9ibϋ7,n2KGFL&(ՙLULQhE IJJL5&0hDڷ ?? LxE]Z br v8Q\Rƈ169PCdO@d+./%Dà"z˥kޟ(@T,KUVf&N dJo$=}i753qz^F?It߲3 7 Kt\Oa]:O\#[`c/cݨ$L@9p`@k\66jTC1f#*86Q[B1pי@4 |3M_>T@61XX!9^]$uRt.$dP*KH8"AWʼnF:eshA6 cNP; KbC<W(Hr%˒UI3ؤGgQȢ8GmܟL.>;'WԹ~߮߾b.O>v5DN cxiJmnNq>?> $d* 뷷N3xhJ6H8FxhXN]I&KD",Z k |*;+ 0q pzqU0 ɷ~.Gϱ5݀m̊ 'Nʪ*12 Qi6S 8Bo'vVٽe_lح] X?#AVAYoU 롊Ks+Z8nyƑ:_kY09"5ȭ"*+_y*WBԨu6߶3*̐̌1 M`䀘UeB lnz@/ L7Wxn Z*T2YYyFSwV"]ӁA4RGV;;q\O3=`kU*șfn4ˑ\rS[a9]-W$1i *M4AƠlדа?By2-lJh@]&x`Icl.t8H3Q19TPMK9M(JQ~+pz׊ª<5Eh Z}T7Hms&JRЭ]ۈy 0o7? ,U.{=Tzpg dzX^&j(DNBB5'v8GxroZHxS2oTR] TT^}Cg[7*g+m@aZ*\)&m%h%?^˽Aj90fApʺU-^j\9A_DγbW.ZE[kߪ<6Dv 3clM\eŸV#YUԹB9d76qd3MF о/J!~27 M̨v\Gst5'>/W( `~9 6x_ 8}_ Ng'@~y,䌧\_Gnj[߸>1(h󰠒ՇL=j?1+SYft,5-[/)vg@3Bq&'@bD %8#GHG)%&ABƤ(11!ny"#E-Ko+o6HYyibӽeRF^ײb#I1@+05ɚwRH˵wʹO"뚉KH1g3,+'\5Nr@o&፽FyY.z.9.؈CuoWn?D&6h=f<ۧ>TpiЫVczէ_1G{mf EACs*3˸EN:8 %:O̸[FvNztz#]& AoNnx6BS9ax'їdK՗]W}9[(ZQDvoy[<˱qm,:/L7U?=aod.YirMP{|2c .0; |8D9ɐjG <lj"rRlw*n-~# FU$(pb0(EiK1*=]RFeRU'q=Ÿd)BOTyF/yYHvoۋGg |n7I@\.ɸO  e$ ^ݡhI6*FBv;$&yA.E ϔ|vzk8I'dڪꃟ{nہ$\t,ץ:G&/![nmz_}(544P1)IᦈDSR-:&FF&$ y6USp+ݙH`B/oh+Ï68rNaן W@qS#}Xj Kn$lؑа\f#iWsZN7 {&(*JV(BCz& Qd=lnDU5 8V}k*2'iqڜi-N!uإ4)&IZL&daw uŦ*fT4Q]0};jdu*mO'Kzcf[ Z<}<O(L]3u"۫D|IWo& LJ CR%3-a!.kQ[ܡ/yv;6ކ%4vOT}H;r:g>ygw5lwƥS81l|NR9Q]R;QD[Ѥ,~BS!Wn+Zc2FM.-B":t(L~E!#FcL#ƅge )VnRR҄l?ٯ0L27h 6"(Q"(L%ե*ou]F0⺮D0pE!  ՅR|S38Z5MJI7ęb+E+DB$L/š&aF\? 1{v5"SЀN*zF/Y%]z /Jghj5]`^BCr+M/Z^0ʦp(:%L5N0^6Tvv!rA~q~U4}'\Z[㳸#VO}®Ǜe]?Fd+ZŽI>'J*eKT2Bt*5lkqVOYO?oËF=Af~D&BglY9z;xqfcpxNH׋ʛ&4iT?3Wp:T(QG~**Bˡ/+/jM`5?]75 iIi)/cT?C_p񻐛B9jJKS281bwd2YlF-UI`ٞ\e73vlfJ@^tF$MQ7btE}t$G iҽJ+RUEO)γ9bNޔbTƞ 5Gn3EǧtfUMSYՄ+цoрu9G-,笗+ QS EJ-WJT9YA}pd(vMBt=o2qT.mz 58`la?]{|ս?gfvvgvvgL2dYASj(Xȋ͆7*>ТVڂǪ՚OmRcr$w~3||n?_|̙;'⧪#X_[ۧq&Ddf @w >hO хu$3/;#WvOLglWkDaiMSz)C#? 8m A  ,Rf !T.b7Lä;e+h*b".5Ep?R+Mꤟ~ eծBe۷>Ay`Z2я5#k}^1"`aĴHz%&*CPw_O8laG?4B%ἚWԨL}\#"zw̓qܽE'MaDY4f7SPjb:%(ǘr] fLUC7`yzT&:]CpZ:ԹWsUIqjѲ;hIUP[][3p`֘UU \呩Uf??k+|劕>] qB hD.)]v[n{RN(^-/Oα_]j־?|Ί .+xwrʯ_?s’%U&(+?YA_ Z E!遒NJ+C֑>2HN@xۖD<3Ud:^-ǃP$)G@7k4k?nfٳa)Ä>Vi _T1USСƨa[5ZS%mM y+k2o熵,y+ ;&z6Y Zv.P[˜ȓeV?e"nlw=cmoٙ۶q׆ƫZM鿓N;]{>]ė18 $g'g/m9L >UY؂u\gGB5Kqyy %jbe%.%'׌ItP~\c ^.F8 };)Io2BJ^-0v֯$3uaTpbfrMQ_, 9o3ofP?jVXDȤ pY 2t5qB,ٖ):qDzIخJM˪ / \F9}pO_p4p ?yC#7piF~ :y0jgXɱd6ni/ JHyR'5ޖ$9O*p$)- ָ{}B0(-Y#.'A0|eanMYZ6!cلL&E`J$3 130lԆ2Om600 |x'>蓓L. )8wqSfTcsָWN21c&U-ya>\ZfT,4Dag|a؏MR6Q~~ LFە;q %l.2\'k.ZAGJG e(H&a W]'e1G| ܬ"Z&Wџfs / g$T?D7̟m3PV:ƙm3އX- QvQSU\v2oB_Sx@lCpc6ޟ 0lxMf!tl+&Dq27Ϝ](CSk?r8M2sIaS?(盛x?Z?b^yeҲRhsz)4blll'k_~oӣÇ[v<[4:^7^~UѤt\2q]}C z*_U9nV ̉4ٮSG+I#>2{*x*!P/ʴ2_P])іi?.','N;q OT OtdA8[nd&koj'3pěx.H8q0x˛5%H/ȼeo0κy2FrA.vY۹޵X{eߩp=q~yz4iyᎅ wyt;|2[z Qࡋtj-j+j!-&$%9ov$>6jb/Iǽ'\Y&MD`2R㘂pKݳ^[:5K }uؑ4tN'n7hsf1bA+[,p]~{s7̮a0*4}? T7V/͑f.ŶJwK7>Qys#`ppVErriܾ\YXg[')E*\,0pdo-j-I +?[r}'G,z_ybPubPsh wn2[q\^9atB 5n=:P^+t:$䅾B/o| IxYu Qqez3 ːξ G35Cu3Ѻ¾(ekgGa*fE`PWUS! nPl $BlzTbOr4́HhVTaj47m6d+3rpoހ6kf6~-`ӉKr ,"</zH%\\/<; "$VPl-)dL<-6Ҳ;v,"faj+8vsؽzM[7ƿ{ӧ޳x \JOr:b_<7Sm3QW]q֖7 .ZhJA4#VOߺ|oiG\ y[dLLOqȔ'~M*SeX$xH:qYm3+>n@`gZOXE+[p;`wu.\x _;rI.m\Y #3NWb#ML,B9K!?IJSǴS@4jdžyBIU'3pI؅KwԼ,hrkH \Ñb ##<"@>-B^O7?wO9#_.!Ey>IȒ%X@TjL̇M0g,26m(+CljQK9ESe@9VvLlh@VFSdd$Ʉ8pL@dpYl6vp*K< UbD"!ItFIX,PBҹ;phy9j6u,1sɶk.< s;!MÎox)hφy?Fn^ӵd#wRA: C7çR(pȉ:D+R ՉL*ꖩg{:hׅysZZoA&)4Y:{=C/:_NoJo8ޓnSv]j.2ir2sYD_1&h,NJD2rЄA3;|bP`1P_LUz1'I'דoX:oBrIm&#ob'L,;L_I?ZK[&ibLX:Ml! Pm$(FM,b/F-S[k֓E,?؊%FC@C[;L 411v@Ch(h(h(ob h(C,q: 9"V|5b'bͱ۱ yˏ GGo`!y".".e)#qaYrV1!ƹ8'NH% h YK>tNI-K`vm$֨7I;"([ߧI> jok+Ԝ8 ߶5P ]W}VMd)cZhLT-%*xjhqBV[_esVy@$q>-ƹB;8PZﮁ$<1Jd1|2z҉؂mř0.L'++|ޒq4kn±0HcNYiΥ߬ƫRe I" õgdpCԑ&^FVXXV,SJ)mҨii<`K{֥8$n`L%֮M]-Ř:c/]޴ڶmkﾐB`P[Pk@vg+ Dq s?kkkkk\`{?-(3_ C;Vud? $a0K\.VhZO ٺiGZ/ > stream x]Mo0 CEBji'q؇FhbH#D!=춓v 68aZ ?$:plPZZpQ:b&p7S Boa ggg OAn%X/lt1?0vu$ koXm+}^e=bM#& ^_ 8Q*~H:-Kr[T'xT"9ёhEDGxD QpuȨCBʬns,Jys SRT7'er``["d/oc [XZBix3յ>5 endstream endobj 22 0 obj <> /W [0 [750 0 0 277.832] 10 [237.793] 41 [610.8398 777.832 0 277.832 0 0 610.8398] 54 [666.9922] 68 72 556.1523 74 75 610.8398 76 [277.832 0 556.1523 277.832 889.1602 610.8398 610.8398 0 0 389.1602 0 0 610.8398 0 0 0 556.1523]] >> endobj 28 0 obj <> endobj 29 0 obj <> stream x|E?zf%FB=  ,;TT$I `EDgw!ܛigʙ3gd!FDVF叻qĸ/:hp,Q7 ~čETA97< O&b'ucr$ܝpߨ'Q˓G14|JeOi{om Z"i;r5oFDt"ګ'OtD7']}qw}nKڈ%׎!E?q:U_% ^AgΜA?6"'89% RH!XxQ6'P8P%R: bGJ] t$'>;Q~ h \dW(ϛ:1Dc/i.B[,*!=va"mA?e:Z#ȯeM*%Eeϧ% ip=ңBg ?A;psoRcc)/+ >'s_ RP_D<50n2Ǎqo1@!'1.`O5?`Jc.xŸ} @3a\ENtC?g "Kits,ݪ.ǐN S;ȯ_@/0\w)M  I ='V : <([ ceOaSm:M`yIَ@`'C`ֵ$h} ڷWMB7M]'f teY(+3LX/n RS5 1NS J}0ۅnXwA(O4_з r&֣XZ`5 $SסAy΁1%Jwr Bo 0y/lX(s6h{uKL)^(}Gbo*3'Ο'@ü6F"x5c+H^ L9ZV +vrQ䆜ґ>@شQ|ݫEQtm?~t.iP_S3*qXw[m||}\,a6W߉&-Fzڼ~j1{ނu~ _GaㄍvN}i`;>6.X@>ulؑmR4mZooov YR;Zeb=R!'/Q%Fb,7с 14 #UXw'K^K_|=TOQ/aq"*7?d9ᵠK(Lh2Ӿ4e= k)Ϥ'MT!J%k>^F4A3sI$xk|gNH(eXdʢΔCCNC eʯ;t -3a~x lk]M~#1D~7':%Eq1Z4ÔG:FQBG0?q :$ݨ9jn/c˵zY:H>")oX/Zjn2vhlI~W]Q8I@+?T` 03,u!zD>&ւ*֐E ;^л3$C\ʒNPO7sOP;\4o\ iC8x>)WϺ ۂǠru6-}D9|-T`5 DW5C՝_W|?},J @zX?^GS"Vȵ7S )4;`/|/{P5\ _E`0!D;7 Ju๰3/Bf|Vi2Yt_ } ŕ,zX}I>@c h KPsoLqZЖUFk#v^^9#* tW}g̷i.Q /;=Hp@,Fb7:{q_x7?o@}~~wٶ~K~}AПQJP-D>{syn̹_L_Cҿ9@Mm|ip>6>kh,'|}8 gSj _'YNѤw 2-Hhpƃ|Fg*~ic?~G&7`1g3XoA4Zw-Nc}kSl-edsTn~ ~}9Yҿ qMe !31% 6ןk!v^qΕ~;_(y7⡈'(Ik:_]|F_ի5f+ͻOH_6pbqԸ1ż[{4,%A i'ųӁZ,hZBIs]pZM))py~JWjDŽ4GTm9! ^e6ɜ%4 ,AuzPxLkFKGzDwhد*d?qNsc_[U ]!gC{mPO; 8 |]{i={~{B6bVlfݔqۂ\ &;*3B 2 JRWoqgG,^9_>yD?es=b~B3{=]m>3;fޅUG}!o.k<|іJK>~t3AzkM < Bv㠣m Lj>zcws롷ܼ"XV/LF]h#iU?qw]4Hl .#HD2s{$t3, H!?kcWЖtA#HKc _]?.=1Wojc =׳191HոO8>'O7nt, 8FoʍsP 4cq`)D[(?Oߵߖnm"Цwu[EѶ {fwߏ-hwЉ"_[s>,?x[ }{3yk\oD_ȟq."l{Ήִ)^|fw*z#Q uX ly7ϓՇH>!⟣%0LV CFWrUѧwizx(į!i4Y_Ns}ȗ U`zl)d RkG4v }:|x)ڈumq5]4ѹR/hhBS&oL.Q 31XW(K3_lG!} OMQ)z=|w#e-@]e> :cD+د Q6T oNa].xш$gi*LWibo}W~%U'$ L qQ}]1?ox{``} l؎tsm"~q } p;0ڟ/|Sͼ녭ุ ocD>ix&ְx/7p`}?/{7G7+ mkkLwXgA^ 'W?3N?ɝ{8Bi~l^οSC  wz j%iNi\;~ݏ;;R@ 5@|$vNWo@w,0'~]_u]v=XqMV|4/cIw^s/xΠ3&I (r؎s@yDs ?/O{L%?yo0F@]uS~ 6*,}+tKNc=)@^ƷOXK|i@$|/Y݈wll_ \N%z&؟i}>_ w/yr6ؤ|{x9H!$pGV|co{ğ~_ߑ'.alC1ߡ((#BqLSÏ~$C.;LE9^ `OF}5w~ŻS=swkD~%"8?d vozaЛH Hԥ ,O<[Cr Pn:SHX2㉼.f~m\J)!O~7Ne\(2#"Jo+;JIr`&<|j?|NAYHj4ݥQOJP#޿?ynxXK,{ͰZqxؤ|= ^`Lxo4Zժh|芀- :;lzpUPNЭ/(o;WvYtx_dx ]fl?|{^ofۻTor[O`yEC ;%o/𪄰tهOK(w7k ܉& =+RԿ?O7\0HJ[};gHew%i;S¦jsi,&Tag b? ag{ɉ'sW;.G~6H!H;YMC+ \w@?Ywb{i;^ߣO>J_ A Q͆h!?)7/sWhy^12{aιМsDckwKt>-xĴmN5qgug_,Э,m=~]i)U-&ISY1].U"וӪF S^$^r[IK~LLtvs^Y9yY+'^W+ŕjw:\]\c]S]KV#cS3[$J|U䇒צ5%4%<%*%>ř=ehʈTHMNtnKwGǦ7KOKM/L/J.}ZsH,}Czu鯤;o22]22qmroj&~MY~٢ϖ-=ѳsO՟z|Vr|FJʥҭtHmGUeJRKRUZ'v]+$J2锓N< !iל9?sr"W8R+pu8/1|JSj]/ HmyAjq)IU 7K-HmFdԪr'Bj1k\gij!5ٮg~pܹ;Rsy {'/;SރPspnϹ@ߧ?߷%:)ToE8v,qL;SI1Nx~?{xxac=<#ZtpUg\-k`Av@m> @ʁ!O'om%Oޟգw|pȘ XK%bQ8գjgl)$]-+=`q+!R`GYԢ hj:=_/Ї+Dhj RhqO3ŲoyĦ[[ 8\u>4Zd=NulZFOgAw>@?\zf<@'i9UޠiFW|Nozޤwmzvߣݴk{Z@tfMt@+i,qt#MI4&M-L-t|yzFwwtEaƙdY:al ՓLc:ۥl[V1f0 2[_W{fkZ[ֳ i ֪[ԭ6uC}Aݩ֩///oooo層w\/{eB S")()b(Ū~U?R?VgAzX\BRJZF=~zL=Pg> k-L "H-JbX-NKКiZ\ZbFZCϭ_X~e[϶_l~n;c;k;gym8DڙkZehZ5rtk5FݩݥݭMfh3Ylm6Wݯh #:-}#ڣ^K`Ŗi˵J1m}B~:HgړjmV{J[6hOkh5Ijg笱8k5̚hM:.k5ŚjM[3,yEU流2by]_//{rW R__֯҇UClksk5ךgmamiͷ~g=f=n=ame-}HXDT_/ї+]+N؏;~b/Wa9,;ꙗ·"8Uq۸Paw-~/O[Gߥ{}'~~P??׿пԿҿֿя {~JI?3LšgsJ^%RRg *qdȆbfaX aS$)NnCq)JjFiDFkFH4 2#ծFidFs#UҔt#ha4VF(4mvF{QP2NFq(6Fj݌2)OFwfbmFqqm}F?1h\nT{aT++Uj 36#5(c1Ƹָθ޸k37$c2|/C|g0?%|)_ƗM_ZΏ[W[>e]g]oݠҝ=}Lit g'FiE!(*)?HI{OϤϥ #?g W:(O[n[V ?z>$&qId~\R,%W)7(J]DU*$r2LQnPnT&KʭEw)ӕ{,e2OAeXY*(˕ZeQU6+[mNelVv+Iy~J9SN*?)*g=|P5LPccjS }fYjs5WmKmPmRgx]RIWejwzSTLV>j__T/W+Jurn#Y$dGRRlܦ,[-oK2lY\[VdsJmmmm+mUQ18;s᳐YEŢZ4n1,b-!Pf DX"ٗ+W7w|uu=ZZ?~lgˏ1뫴,UVHfz}M5,b}9g>l.g}] .*~R̎XvV*:zVlV갆޵g[?GDOBVL6-` &e[ u:Ifbjfna}Zg}ow==>a9ɿ_ohE~Je\RZVʕ>!J2ST e.tw2Q-SKR-56ߦܮM }<=Uʑr A;EEJ-J?Iq"Q/| 6odJja!ȨظfINWrJjZzFfVvܼ-[.lӶ];]Թ]ҥki祗]ާo^^Q9+_5j( z׌=뮿a7N8iM7rm1e;{=7csλ<Тyt1-]|V=ē׬}jzillynٺmvֽH/k[o{?pG'?CMw(Mw(Mw(Mw(q;tttttttttttt;wIwq狊:uо][ir&%6KsmVk"KQnԲ*'#gv'C0ABDž?b?tF%%UDEyn.ϮTW-ԧṥ. 3|o''-vTê\++QxyzYլ24=Bυ=vtQ7"H]RGS塾$WǻS|7׬ɞʡ6EҬĹ]q vSHh `7 8g"Գy2ѣPjzR1#Ӭۣ~*<1#=FתY"]{tGk8T8ǔ53Г`ؓi\s>v6mr'.!ۡ[Bbg׺i"i}*q K&w˜J9u"gZ0<{U*4Y3ʣgꈎ6EC~~= puUm۟ <]+̅R^yTw>@BXzư7O0 ZUĥUc"-S(cD8L:^bᏱ vV>A)n'*_܂s|wl;w;ϽZO_-]yvծclG.|>Zn)ǘĸ8MJ3Tqf8*Po)T`p' LLn5qq&8p8p39ƙOG8Q* UG![*(GQrp(G98M78pmrn 78&G>8|#Gɑ|p.pp8\& .pL8par8a$@papa0808qM=%eXeɲ,{,{L=`=O46S4@ցu3yL^8ALz;6mtIov&2LL::v - +N@3CCqȡZomQ۩)U{+ԍNU٨V$M;*cϩ< `g*hv ~ y;ds9ٜmloJ ~1MKv8:Yۖٹhi1ꌶZO9ǁMj.PLk@ہL p&(Z< [yNh'3 |۪3Aj3{<_9Yb͔)"fn=jW~O6T;v lrEu.g ,Xh?[о΁(֧ڙ S!J7GCfhz+RjHJ(SxR=TANna2s['8 U+Ne(\.6G眫g:.s^^Lj]8wG89rNp^jT*rTHv^^kvytvpmzm빐oZղ0wsmvE뤥j)ZECmEuUuY;oDS6.>ͿbVtN'B{zzꮦ\_2 7Jj I=wYzhWTlbl^%R=|F-'I37a 1wJ\[9CY_|T>s.6 &zWYX)_beOϝa n[x [q<[_.+DbLAPLB.Q (9;% r;e2,vDM\J7\ft}f}Ԡ 422R.V!JTٱl"Efά<-/IisH-]( Bψ.9դ)/n#*ɣb=ӆ\L dT zCGx&(LI-umj_d,[n{Diu+wnCK+k*JmUEeE Vq_dbVhDU.66Z}y&TvOkJHY(NɱS֒5cK"+$Ddaq;Ȋ)9a+[r 9, EKPOO>==8 UM?fv,u]O4߆%i_LI&Mr&4Ӷzihi-idm2n:dlhNrX$ԥJĚĂ;Op7U4tq~XӲ*hu|rZiVAyۭL_ͫ\-ꖫ%3!('VBhff+E '2g3ga vB fO3o}RmR̜d2+\AHJLfrX*hW"OP,yj@,vb7ZQhp6R==}Zqfԃ(CsdQ*.:OoAtT*}IȊ3]_MC#><@ voh5B}ETB%}9͑+hmcjhxH)4> iL=^C!zK]h &E-ʕ}|G> ?)|}xo|F[[{ 癅mg/*ʼ;}! v"I?)>7S?*Kd.>EZ`I+gcX0[s?un\dx{z~ƙ;.[i*d]K'9X{6=<0; {q|NOKҋr|KTW ռV{>}=NϠ2HNhœ>@g߉ bW l{=^e0J2Sx'^V!OSOgYR4^zLH!g-Vroy(+z%HSҦ7?%([աIBiY[a:Yg,XOv]F}l!{-a35=~|(\,~7G|?HRJ! n&JSi[@:"}+I$VQyrr=~W);:=rNjLmQת_hV+fjj?X3=w5|a &uη.,/RLkZN֎MyԏM"*.3`?z@w*ցQr.TK0{0wM5 +3~ 7l*c,unzVϭS;˷;UB񎖗_°h-(+khEJt lIVu9 tǷwXzgX.;VbEԂ ux񿿱W?TG߱X N(:Ye ҞNK_@-}G2sGTG+:^),afw dj [k؉+8F: 9A՘Y Rj7cwky.Q"X:}i~.WT6߭TzNcR D|8z{1>@E(QOQ7E>`ax'低ަ0'nyVz^x=QX!m "?G(:nÁsD/ rD1mЄ&4 MhBЄ&4 MhBЄ&4 MhBЄ&4 MhBЄ&4 MhBЄ&4 MhBЄ_/SKiYμVˋ^,eKYb)6KQ}Q/ꋨa9|OKK#ιsne.N|}ĕ&6Oي:U~UT&+)ciNR-V񷫜}^56m /;сw*8>Bj%-h Zʰ~?L$OG.YU5rK}u$93nUWiǻVn sQ>qk,; F1xň;<TREydK';Ky~0eh}RϤd}}힘 &DŽ'defg%ݭWA1m7lvn#DjH}3cH,YjC {`ˆZn*c!H ȝ.!̕jqZ-ϩI?w>1oןUN0!U> f6ak%()j#uAvmb>ewsxm2}Zz?un+Ov5|͆[S޳޳= bs g1BPnRwqdL!)n\VRS~Pi 4Ks輇TG?1n[~c@1 ќLol=S~,=<M_e)]!]V5.5T"ՖJ"ZԚe3g{*47)hJ.hKXmre-BȚlT[e@ȀQ$κ&{%Sk}kGIbʋwܙyj=-%-04k[췦̴JYmYc_aX Y)ZJ.8WX66vE,GPL` yugb^zĻ 󙛕*6c:^vwpLknĞŰwDLaLO-3#3sd_qyd~ݾCI͹s|9_9_tlΟ>J<;C{)boVwhx+DHbC%V rʻpGutIi9wM)K]my*B+ GGGc%)4L45*2&Z*r%\tiz)b4ﾈ;G̳{~}l[Hoi-ό{y䩷nҡ0w̞rZ&z[ȝ) x(-^o$F_нYCaF۸3F]qo¸WoIx=ڣոL5;2&~/_>v~irii,|%M;䇄^(ёI=QNLeɍPrkŠ |&ɚny&EIQ"%HkR =ȲW:m+lic>C m Yaռ|X!1P 3$flztIpCƟ,l^?'00 BLꖉl| d OH,6<Ω XG)X %Ĕ`dbjB;Ba/ e m6—=twV&ub;.Yc{gW.]EZ;v=6kU4gtW>:y/,4 ]HeGTVWF >:[^sأQhѨԈ. $,6ڦu o.VI)>ЙqkDo mP(sᑅ!"%.ФaL& 0 )FCD!nZJ8'990E,&+朤dGmo ?b=vh ֻv)<ʢ"\s >þ}ݻuNuЅ>riU[^a7%ޗmqOk%X*OYkZrtEhede굡:SV\l=JsXYmvks=3$:&*n&&49S̉ 3TciVs/2iB"Q 1/Mrf b`b֌XF\\|X+Vjj_QXo :N8 ֟ѿ%m"36l o^@9x| }tdR\Ic蘠-l5!<Woa%zbڥGا}t0v:im{swW]3oƨweJJnzՒ ݿYYx1E[Z'W.۫MPdD:.aДh%W SaR$gBfX,Q?MZ2tJ+h0 SV8?ve,C,?UdFtlQve!Eo&pA@U4rtc&G,4͟*lCNq#Wt7՚AmJg[n?tukt2f)vfx:cqPF1XER5 Pbnӻh0jk.&qz`\ƁV8ỏ`q#,Ndﹳg? A|Ê%qy"LcѲST%qkUeqH>!V^Z/}z=(xXI֣A]?k!~0R8,>jp\xwƿ<~N'Od.'.fN1T^6+O7e1,Kyc~q}ws^wEx]Ak DQ^X$ʱD*/s ʻ+$GɢkUKq 8 |͂,\d ЛTc$ b}!eSS,n+4bAuce!]89i!U!r oM!͞/M/8Cw7=2@di`@%s,͋{hi-8b1Pf6`ĭ@o~űL+*$Ãrr̞*`OP /14|ao9=;Qg3=3b#`1'z9fbK5a=BIddS&Ew:)!ez`@P" Sm ƻ=&HBk_A4-m,#u2&5BQOIh%@EoVN:Ku#묛kd_:4!@cs?}"j]=1Jff.5PԝG(o p]Iڒjh%%_swl$}FN-Gvgrca gG^^(b A6]kȋ"͆@z [E[\2ǘξ1F=:)p?5+U;:֡gq/kh7>x}&*שּׂ`# R WVnڍZj.ճhh]'/.WKzn][% Y7~;KRvV͞&Yj(syNE]/tvJ]LSnΘeY5) Q2H#St {c۵)R/{ ݨnT,S;}*Rxkt]ۡ6p+Eױrz'%5k&}*.s5i730˴|A]^) =6<Է`W芾N+e,rvѥM \*%b8&\2-Ί23H,2`:ۨd]Z V1[Z5.iJtIw c,<xFѣ cV,غsQˋ]qK* 3B1BiXMcv㒊AfûQxiҷ;ެ7N{'c' n{pu3JAqrTv8AS52#m5nrx;N#st0+X7Hoǯc myT!yL>!OS7gǣLp dyI Ji{D.D_g5Y\%-5I}Һz=]eI( endstream endobj 23 0 obj <> stream x]n0E /E(i%}@!Te_3CR wDM{l!"miD9R#bh_q@vBq 5,-.dOdRm.MZYo8RǫƸנ`'v/ @ endstream endobj 1 0 obj <> endobj xref 0 30 0000000000 65535 f 0000093979 00000 n 0000000015 00000 n 0000000063 00000 n 0000004223 00000 n 0000004317 00000 n 0000004419 00000 n 0000004517 00000 n 0000004621 00000 n 0000029089 00000 n 0000029686 00000 n 0000030180 00000 n 0000030566 00000 n 0000030704 00000 n 0000030838 00000 n 0000002443 00000 n 0000024067 00000 n 0000029583 00000 n 0000030977 00000 n 0000044943 00000 n 0000045319 00000 n 0000071481 00000 n 0000071876 00000 n 0000093596 00000 n 0000031335 00000 n 0000031555 00000 n 0000045830 00000 n 0000046051 00000 n 0000072302 00000 n 0000072529 00000 n trailer <> startxref 94035 %%EOFbeangulp-0.2.0/beangulp/file_type_testdata/example.ps000066400000000000000000000002031474340320100227440ustar00rootroot00000000000000%! 0 setlinewidth newpath 30 64 moveto 11 { 5 { 552 0 rlineto -552 6 rmoveto } repeat 0 34 rmoveto } repeat stroke showpage beangulp-0.2.0/beangulp/file_type_testdata/example.py000077500000000000000000000002151474340320100227600ustar00rootroot00000000000000#!/usr/bin/env python3 __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import platform print(platform.system()) beangulp-0.2.0/beangulp/file_type_testdata/example.qbo000066400000000000000000000152271474340320100231170ustar00rootroot00000000000000OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:TYPE1 ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE 0 INFO OK 20130929011247[-5] ENG 12824 L998080235121018 0 INFO OK CAD 003801010 011016298027 CHECKING 20130626120000[-5] 20130903120000[-5] CREDIT 20130626120000[-5] 105.07 01001010110020526K011W9T4J334 MISC PAYMENT QBZUKU VSLINMR DEBIT 20130702120000[-5] -17.00 00901900931220793X01121QVSF34 LNWR BNETVUEKI XZQQJJ NBPS DEBIT 20130703120000[-5] -4.90 90910021021231794B0922MIKTX14 RUTFOLN HJO ATM 20130722120000[-5] -400.81 99191029929249833O900235RW47W Withdrawal YWP MI --- IT705538 DEBIT 20130801120000[-5] -56.09 01098091941031981O2108VG08019 POCI OYYOKLFWS SKRMTB JELM DEBIT 20130802120000[-5] -5.00 60892802002122920U23369T72FSV PCELRFA PYE CREDIT 20130805120000[-5] 537.43 80900900130241904R191PVXD48YX Transfer PNB TMTKERXZ - 7269 DEBIT 20130805120000[-5] -236.79 09119010030129796M1105H284G9C Transfer XBQ PDDNOCGV - 7292 DEBIT 20130903120000[-5] -38.28 60021009939131914N1112UI45W3Y ZPDF FCVSFSQTB VQSWZE RXLS DEBIT 20130903120000[-5] -4.11 80900011029239092R902HZ5B8V1V CEBHSFM CQM 368.62 20130928 469.82 20130928 W098988027229041 0 INFO OK CAD 790012098 122384459041 BJNZDIF 20130702120000[-5] 20130903120000[-5] CREDIT 20130702120000[-5] 3.76 09918208128232691H10066Z54E76 KHWIYVV GDGHQYLQ CREDIT 20130801120000[-5] 3.63 80180001808341702Q900O2409E38 DEPOSIT INTEREST DEBIT 20130805120000[-5] -449.42 92898928800022624R0134088U0YC Transfer HGA ZGIIQCHU - 5468 CREDIT 20130903120000[-5] 4.21 79292910822930892U9121H26ZU75 DEPOSIT INTEREST 2468.87 20130928 4469.99 20130928 D9935431020024575686 0 INFO OK CAD 2542393734236684 20130720120000[-5] 20130803120000[-5] DEBIT 20130720120000[-5] -34.69 98900821118339830P291H17G1J6P KMW QAJJESZTLJV AIC981 KLDSXBBC XQ DEBIT 20130721120000[-5] -20.22 08920030101211833G00911K3UM66 GQENB FVLBVDSG BP DEBIT 20130723120000[-5] -66.56 02120210239221612Y099G1230080 IPV GDIDJXEZMKISAS FUXVLVTN OU DEBIT 20130724120000[-5] -48.63 19229932930218512Q0195G5LKG9X YLPSYD BHFWCGXV QAPDM XJYBNEJC KF DEBIT 20130724120000[-5] -08.96 98821190840028743C802A5455Q9S OGJQA RFMIBTEW HB DEBIT 20130725120000[-5] -5.28 91118099248332546B119638E49F5 XCSI ANEZ/QOFSKU QGSB GDJXAFG DX DEBIT 20130726120000[-5] -69.65 08999909831339518J011I0270212 HMG IYPKLK IQFRKYXM EX DEBIT 20130727120000[-5] -6.09 80018129231330939W910CS443Y5B BBLC NZUF/OUFGOS ASWU PGXZAKZ YL CREDIT 20130803120000[-5] 126.57 89008210112118684B8297P72103Y RMJIOOH - ZFFJM JDV / WPP TFUGE - TWYFZ 0.00 20130928 0.00 20130928 beangulp-0.2.0/beangulp/file_type_testdata/example.qfx000066400000000000000000000206141474340320100231300ustar00rootroot00000000000000OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:NONE ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE 0INFOSuccessful Sign On20130929012330[-5:EST]ENG20010918083000Yas Zthobhkd Xregh93783120LJXPVHAFFWB3288533098615-2-79007630602164240INFO20130927160000.000[-5:EST]USDerwhgxceqz.llv88584720130828160000.000[-5:EST]20130929160000.000[-5:EST]9330189228209212249188631VCB20130830160000.000[-5:EST]20130830160000.000[-5:EST]Jdcvj xz rs gjjg vryrt rp ixstroq jtczf002828564CUSIP34.3434111.79175.63CASHOTHERPRETAXBUY0169012510129243631021242MRY20130830160000.000[-5:EST]20130830160000.000[-5:EST]Zwrcx ze ea akbo grrwl ne jylhabl cacjs599884005CUSIP8.5386548.92562.81CASHOTHERPRETAXBUY2943499296319002323880850DDD20130830160000.000[-5:EST]20130830160000.000[-5:EST]Kyata zz xb cndi oewto dz yrcwfuy nhskh012059574CUSIP19.36611.50182.42CASHOTHERMATCHBUY0960201409128219633283042IPX20130830160000.000[-5:EST]20130830160000.000[-5:EST]Matqy hf vs zvgu nvzal lk rcybjer foctx380892005CUSIP4.6841569.80301.41CASHOTHERMATCHBUY3262113318349140462189040MIK20130913160000.000[-5:EST]20130913160000.000[-5:EST]Epprp eg bo vmtj rgtxt za gromfnu xpiag922055897CUSIP47.70690.71192.60CASHOTHERPRETAXBUY1960319307319200362125062CFM20130913160000.000[-5:EST]20130913160000.000[-5:EST]Xubha cl ci ogge owflj io qphwxgu kdjuc117785825CUSIP1.2323.67404.52CASHOTHERPRETAXBUY3932289101112833642282030AXD20130913160000.000[-5:EST]20130913160000.000[-5:EST]Rvpnd gz jb zboj tpufp tu cojvoyr ntire033957506CUSIP97.25220.45201.52CASHOTHERMATCHBUY9352493380349122444221482DWA20130913160000.000[-5:EST]20130913160000.000[-5:EST]Ahwkh lz cw muvt xjxnk wv uojinvv qfaxl318685906CUSIP5.4253052.98391.52CASHOTHERMATCHBUY2352329428191912250211761AZV20130927160000.000[-5:EST]20130927160000.000[-5:EST]Chpjy xa pu kbdh bfnjk pt bpbjdgb neybs901155666CUSIP37.10932.48206.64CASHOTHERPRETAXBUY3952191410469239670114345OFR20130927160000.000[-5:EST]20130927160000.000[-5:EST]Mvwcq we nd rbbk borqn al mhkiwkp tambd178073607CUSIP1.1915531.5476.82CASHOTHERPRETAXBUY3329014908282032683678171KFW20130927160000.000[-5:EST]20130927160000.000[-5:EST]Zrdkb wu ed ixiw qvvxm nz xevnxfq gfjmj833117594CUSIP16.0422.66074.49CASHOTHERMATCHBUY3231310407461032543881155LQS20130927160000.000[-5:EST]20130927160000.000[-5:EST]Mltci zb dr mkeg xmpgm ef foulmqh bpjue210792737CUSIP5.6677360.4373.42CASHOTHERMATCHBUY06424419410848024252919669ROW20130905160000.000[-5:EST]20130906160000.000[-5:EST]Zwqujkrxmq Wckkbyl842929565CUSIPCASH1.973OUTLONG22.69PRETAX22408411481280810231222931KAW20130905160000.000[-5:EST]20130906160000.000[-5:EST]Jhzpojlfaq Mtrlnjf701746774CUSIPCASH1.995OUTLONG00.46MATCH03618381388079013866095386NZM20130905160000.000[-5:EST]20130906160000.000[-5:EST]Wynvujjuok Ecppzsp497982895CUSIPCASH2.15063OUTLONG38.72PRETAX44346292481140211947882463EFV20130905160000.000[-5:EST]20130906160000.000[-5:EST]Avcsyvszfj Gzciwit417862735CUSIPCASH9.24172OUTLONG58.84MATCH0165393227021112149898978YQB20130831160000.000[-5:EST]20130903160000.000[-5:EST]Rlvhw mh os aqde mptnu at reofxry ftjkj811716965CUSIPDIV36.13CASH0.30002.69PRETAX3884013528141824544812942SNM20130831160000.000[-5:EST]20130903160000.000[-5:EST]Xzxrw cr ek hxwu tdnes dv ytceddd npgbu132036684CUSIPDIV7.73CASH8.7190.47MATCH379756617CUSIPOTHERLONG211.46753.233073.4120130927160000.000[-5:EST]Zorsa ut ha wccx gyrga ju ocartej ezsdvYY122137597CUSIPOTHERLONG0073.60728.4521618.4120130927160000.000[-5:EST]Vmbwf jg jc lmqo ansms ll ooywgji xhhmmYYPQBCOQSJDS PSO. 323(R) AVQVNWR MKOM208.19.81.21.11.21.10.21.29.02.1929718966CUSIPRgeyznro Jyoui Fqnw Gkuigd Xktch Slxc Jxfcosqntmoqa Smvi CccwicPZFME894009.4720130927160000.000[-5:EST]Immnz cp il bcbr fjmub aq bulhjsr jhjvi3.2920130927160000.000[-5:EST]481694617CUSIPErcrqfeg Tecgh Fme Fswsiw Wvgj zs Dcwbsst O5BHUHXS42934.320130927160000.000[-5:EST]Kpqte fx mg bxam nosee zk pbbfqyq pbbbf0.8120130731160000.000[-5:EST] beangulp-0.2.0/beangulp/file_type_testdata/example.sh000077500000000000000000000000441474340320100227420ustar00rootroot00000000000000#!/bin/bash echo "This is a script" beangulp-0.2.0/beangulp/file_type_testdata/example.txt000066400000000000000000000010461474340320100231470ustar00rootroot00000000000000======================================================== beancount: Double-Entry Accounting from Text Files ======================================================== .. contents:: .. 1 Description 2 Documentation 3 Download & Installation 4 Filing Bugs 5 Copyright and License 6 Author Description =========== A double-entry bookkeeping computer language that lets you define financial transaction records in a text file, read them in memory, generate a variety of reports from them, and provides a web interface. beangulp-0.2.0/beangulp/file_type_testdata/example.utf16000066400000000000000000000015601474340320100232760ustar00rootroot00000000000000====================================================================================== ledgerhub: Import of financial data files for double-entry bookkeeping languages ====================================================================================== .. contents:: .. 1 Description 2 Project Status 3 Dependencies 4 Installation 5 Download 6 Documentation 7 Copyright and License 8 Author beangulp-0.2.0/beangulp/file_type_testdata/example.xhtml000066400000000000000000000010171474340320100234620ustar00rootroot00000000000000 Misc
beangulp-0.2.0/beangulp/file_type_testdata/example.xls000066400000000000000000000115011474340320100231330ustar00rootroot00000000000000..2013-11-28 11-44-171.0FalseFalseDateSymbolDescriptionActionQuantityPriceAmountCurrencySettlement2013-07-04TMQ2917JXN TAAW DNKI YMHUDX M (9086) LCXU AQCDEX LAA NBAPNEOIOEXH AB10CAD2013-07-042013-07-04JRQ29141081KQO LEQ SPTV XHPL TQ A (1813) FAUA ZNCHPC XSX CDEZODXAUEXH AB-10CAD2013-07-042013-06-28GGM09202998UAK POW VMMXWA GXLT MPXZ FSIC NCMW OY U KO (2139) RE MR 16/40/01 JJLMQUUQ @ $01.3473DIV F60.5460CAD2013-07-032013-06-28TGQ01172289JUK KEO WKSV NELG MJ U (1116) LH SA 17/47/34 FFHPQUAP @ $5.5525DIV F60.6850CAD2013-07-032013-06-28DAQ0215ULE GIIX OVSN IQFSOM N (9887) BC SP 87/30/03 ENJZJZYY @ $4.2416DIV F6310CAD2013-07-032013-06-28ZDP3898KYH APNEBF BQSW VZVKJ IYYE SRMI ZH F KD (3202) QD XO 94/07/94 DMNDNDZY @ $99.4576DIV F6460CAD2013-07-03
 beangulp-0.2.0/beangulp/file_type_testdata/example.xml000066400000000000000000000015231474340320100231300ustar00rootroot00000000000000 Spirit of Eden Talk Talk 1997-09 f9bd145d-7b0e-4440-b572-c21501c9fb74 B000005RS5 The Rainbow Eden Desire Inheritance I Believe in You Wealth beangulp-0.2.0/beangulp/file_type_testdata/example.zip000066400000000000000000000005521474340320100231330ustar00rootroot00000000000000PK%}D&exampleUT e7Sl7Sux Kn0 D:\/ xQ] h(!Qr.О ܽ3[8L:t81F!5L\0jF:$c|Šz:1/Q&$ݤinb-t]<.>Ţ&=Ǭ _FV˦>ƅ%p' (%~n>HR ^.@=KmW5;~PK%}D&exampleUTe7Sux PKMbeangulp-0.2.0/beangulp/identify.py000066400000000000000000000017541474340320100172750ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" from beangulp.exceptions import Error # A file size beyond which we will simply ignore the file. This is # used to skip large files that are commonly present in a Downloads # directory. FILE_TOO_LARGE_THRESHOLD = 8*1024*1024 def identify(importers, filepath: str): """Identify the correct importer to handle a document. Args: importers: List of importer instances. filepath: Filesystem path to the document. Returns: The correct importer to handle the document or None if no importer matches the document. Raises: beangulp.exceptions.Error: More than one importer matched the file. """ match = [importer for importer in importers if importer.identify(filepath)] if len(match) > 1: match = [f' {importer.name}' for importer in match] raise Error('Document identified by more than one importer.', *match) return match[0] if match else None beangulp-0.2.0/beangulp/identify_test.py000066400000000000000000000025311474340320100203260ustar00rootroot00000000000000import unittest from os import path from beangulp import exceptions from beangulp import identify from beangulp import tests class TestIdentify(unittest.TestCase): def test_identify(self): importers = [ tests.utils.Importer('A', 'Assets:Tests', 'application/pdf'), tests.utils.Importer('B', 'Assets:Tests', 'text/csv'), ] # Pass an absolute path to identify() to make the cache code # used internally by the importers happy. This can go away # once FileMemo is removed from the importers interface. importer = identify.identify(importers, path.abspath('test.txt')) self.assertIsNone(importer) importer = identify.identify(importers, path.abspath('test.pdf')) self.assertEqual(importer.name, 'A') importer = identify.identify(importers, path.abspath('test.csv')) self.assertEqual(importer.name, 'B') def test_identify_collision(self): importers = [ tests.utils.Importer('A', 'Assets:Tests', 'text/csv'), tests.utils.Importer('B', 'Assets:Tests', 'text/csv'), ] importer = identify.identify(importers, path.abspath('test.txt')) self.assertIsNone(importer) with self.assertRaises(exceptions.Error): importer = identify.identify(importers, path.abspath('test.csv')) beangulp-0.2.0/beangulp/importer.py000066400000000000000000000167731474340320100173320ustar00rootroot00000000000000"""Importer protocol definition.""" __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import abc import datetime import inspect from typing import Optional from beancount.core import flags from beancount.core import data from beangulp import cache from beangulp import utils from beangulp import extract from beangulp import similar class Importer(abc.ABC): """Interface that all source importers need to comply with. The interface is defined as an abstract base class implementing base behavior for all importers. Importer implementations need to provide at least the identify() and account() methods. """ @property def name(self) -> str: """Unique id for the importer. The name is used to identify the importer in the command line interface. Conventionally this is a dotted string containing the module and name of the class, however a specific format is not enforced. """ return f"{self.__class__.__module__}.{self.__class__.__name__}" @abc.abstractmethod def identify(self, filepath: str) -> bool: """Return True if this importer matches the given file. Args: filepath: Filesystem path to the document to be matched. Returns: True if this importer can handle this file. """ raise NotImplementedError @abc.abstractmethod def account(self, filepath: str) -> data.Account: """Return the account associated with the given file. The account is used to determine the archival folder for the document. While the interface allows returning different accounts for different documents, normally the returned account is a just a function of the importer instance. Args: filepath: Filesystem path to the document being imported. Returns: An account name. """ raise NotImplementedError def date(self, filepath: str) -> Optional[datetime.date]: """Return the archival date the given file. The date is used by the archive command to form the archival filename of the document. If this method returns None, the date corresponding to the file document modification time is used. Args: filepath: Filesystem path to the document being imported. Returns: A date object or None. """ return None def filename(self, filepath: str) -> Optional[str]: """Return the archival filename for the given file. Tidy filenames or rename documents when archiving them. This method should return a valid filename or None. In the latter case, the file path basename is used unmodified. Args: filepath: Filesystem path to the document being imported. Returns: The document filename to use for archiving. """ return None def extract(self, filepath: str, existing: data.Entries) -> data.Entries: """Extract transactions and other directives from a document. The existing entries list is loaded from the existing ledger file, if the user specified one on the command line. It can be used to supplement the information provided by the document being processed to drive the extraction. For example to derive the prior state of the inventory. Args: filepath: Filesystem path to the document being imported. existing: Entries loaded from the existing ledger. Returns: A list of imported directives extracted from the document. """ return [] cmp = staticmethod(similar.heuristic_comparator()) def deduplicate(self, entries: data.Entries, existing: data.Entries) -> None: """Mark duplicates in extracted entries. The default implementation uses the cmp() method to compare each newly extracted entries to the existing entries. Only existing entries dated within a 5 days window around the date of the each existing entry (two days prior and two days past) are considered. Duplicated entries are marked setting the "__duplicate__" entry metadata field to the entry of which the entry is a duplicate. Args: entries: Entries extracted from the document being processed. existing: Entries loaded from the existing ledger. """ window = datetime.timedelta(days=2) extract.mark_duplicate_entries(entries, existing, window, self.cmp) def sort(self, entries: data.Entries, reverse=False) -> None: """Sort the extracted directives. The sort is in-place and stable. The reverse flag can be set to sort in descending order. Importers can implement this method to have entries serialized to file in a specific order. The default implementation sorts the entries according to beancount.core.data.entry_sortkey(). Args: entries: Entries list to sort. reverse: When True sort in descending order. Returns: None. """ return entries.sort(key=data.entry_sortkey, reverse=reverse) class ImporterProtocol: """Old importers interface, superseded by the Importer ABC. The main difference is that the methods of this class accept a cache._FileMemo instance instead than the filesystem path to the imported document. """ # A flag to use on new transaction. Override this flag in derived classes if # you prefer to create your imported transactions with a different flag. FLAG = flags.FLAG_OKAY def name(self): """See Importer class name property.""" return f"{self.__class__.__module__}.{self.__class__.__name__}" __str__ = name def identify(self, file) -> bool: """See Importer class identify() method.""" def file_account(self, file) -> data.Account: """See Importer class account() method.""" def file_date(self, file) -> Optional[datetime.date]: """See Importer class date() method.""" def file_name(self, file) -> Optional[str]: """See Importer class filename() method.""" def extract(self, file, existing_entries: data.Entries = None) -> data.Entries: """See Importer class extract() method.""" class Adapter(Importer): """Adapter from ImporterProtocol to Importer ABC interface.""" def __init__(self, importer): assert isinstance(importer, ImporterProtocol) self.importer = importer @property def name(self): return self.importer.name() def identify(self, filepath): return self.importer.identify(cache.get_file(filepath)) def account(self, filepath): return self.importer.file_account(cache.get_file(filepath)) def date(self, filepath): return self.importer.file_date(cache.get_file(filepath)) def filename(self, filepath): filename = self.importer.file_name(cache.get_file(filepath)) # The current implementation of the archive command does not # modify the filename returned by the importer. This preserves # backward compatibility with the old implementation of the # command. return utils.idify(filename) if filename else None def extract(self, filepath, existing): p = inspect.signature(self.importer.extract).parameters if len(p) > 1: return self.importer.extract(cache.get_file(filepath), existing) return self.importer.extract(cache.get_file(filepath)) beangulp-0.2.0/beangulp/importer_test.py000066400000000000000000000012441474340320100203540ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import unittest from beangulp import importer from beangulp import cache class TestImporterProtocol(unittest.TestCase): def test_importer_methods(self): # Kind of a dumb test, but for consistency we just test everything. memo = cache._FileMemo('/tmp/test') imp = importer.ImporterProtocol() self.assertIsInstance(imp.FLAG, str) self.assertFalse(imp.identify(memo)) self.assertFalse(imp.extract(memo)) self.assertFalse(imp.file_account(memo)) self.assertFalse(imp.file_date(memo)) self.assertFalse(imp.file_name(memo)) beangulp-0.2.0/beangulp/importers/000077500000000000000000000000001474340320100171255ustar00rootroot00000000000000beangulp-0.2.0/beangulp/importers/__init__.py000066400000000000000000000000001474340320100212240ustar00rootroot00000000000000beangulp-0.2.0/beangulp/importers/config.py000066400000000000000000000053561474340320100207550ustar00rootroot00000000000000"""Mixin to add support for configuring importers with multiple accounts. This importer implements some simple common functionality to create importers which accept a long number of account names or regular expressions on the set of account names. This is inspired by functionality in the importers in the previous iteration of the ingest code, which used to be its own project. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import logging class ConfigImporterMixin: """A mixin class which supports configuration of account names. Mix this into the implementation of a importer.ImporterProtocol. """ # A dict of required configuration variables to their documentation string. # This declares the list of options required for the importer to be provided # with, and their particular semantics. REQUIRED_CONFIG = { 'FILE' : 'Account for filing imported files to', } def __init__(self, config): """Provide a list of accounts and regexps as configuration to the importer. Args: config: A dict of configuration accounts, that must match the values declared in the class' REQUIRED_CONFIG. """ super().__init__() # Check that the required configuration values are present. assert isinstance(config, dict), "Configuration must be a dict type" if not self._verify_config(config): raise ValueError("Invalid config {}, requires {}".format( config, self.REQUIRED_CONFIG)) self.config = config def _verify_config(self, config): """Check the configuration account provided by the user against the accounts required by the source importer. Just to make sure. Args: config: A config dict of actual values on an importer. required_config: A dict of declarations of required values. Returns: A boolean, True on success. """ provided_options = set(config) required_options = set(self.REQUIRED_CONFIG) success = True for option in (required_options - provided_options): logging.error("Missing value from user configuration for importer %s: %s", self.name(), option) success = False for option in (provided_options - required_options): logging.error("Unknown value in user configuration for importer %s: %s", self.name(), option) success = False # Note: Here we could validate account names further by looking them up # from the existing ledger of entries. This is under consideration. return success def file_account(self, _): return self.config['FILE'] beangulp-0.2.0/beangulp/importers/config_test.py000066400000000000000000000030651474340320100220070ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import unittest from beangulp import importer from beangulp.importers import config class SimpleTestImporter(config.ConfigImporterMixin, importer.ImporterProtocol): REQUIRED_CONFIG = { 'FILE' : 'Account for filing', 'asset_cash' : 'Cash account', 'asset_shares' : 'Positions', 'dividend' : 'Dividends', 'fees' : 'Fees', } class TestConfigMixin(unittest.TestCase): def setUp(self): self.default_config = {key: 'Assets:Something' for key in SimpleTestImporter.REQUIRED_CONFIG} def test_constructors(self): # Test invalid input type. with self.assertRaises(AssertionError): SimpleTestImporter('*') # Test a succeeding case. SimpleTestImporter(self.default_config) def test_file_account(self): config = self.default_config.copy() config['FILE'] = 'Assets:FilingAccount' importer = SimpleTestImporter(config) self.assertEqual('Assets:FilingAccount', importer.file_account(None)) def test_invalid_missing(self): config = self.default_config.copy() del config['asset_cash'] with self.assertRaises(ValueError): SimpleTestImporter(config) def test_invalid_extra(self): config = self.default_config.copy() config['asset_other'] = 'Assets:Other' with self.assertRaises(ValueError): SimpleTestImporter(config) beangulp-0.2.0/beangulp/importers/csv.py000066400000000000000000000510041474340320100202720ustar00rootroot00000000000000"""CSV importer.""" # Postpone evaluation of annotations. This solves an issue with the # import of the standard library csv module when this module or its # associated test are executed as scripts. This requires Python >3.7 # and has to wait till we will drop suppport for Python 3.6 # from __future__ import annotations __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" from inspect import signature from os import path from typing import Callable, Dict, List, Optional, Union import collections import csv import datetime import enum import io import warnings import dateutil.parser from beancount.core import data from beancount.core import flags from beancount.core.amount import Amount from beancount.core.number import ZERO, D from beangulp import utils from beangulp import cache from beangulp import date_utils from beangulp import importer from beangulp.importers.mixins import filing, identifier class Col(enum.Enum): """The set of interpretable columns.""" # The settlement date, the date we should create the posting at. DATE = '[DATE]' # The date at which the transaction took place. TXN_DATE = '[TXN_DATE]' # The time at which the transaction took place. # Beancount does not support time field -- just add it to metadata. TXN_TIME = '[TXN_TIME]' # The payee field. PAYEE = '[PAYEE]' # The narration fields. Use multiple fields to combine them together. NARRATION = NARRATION1 = '[NARRATION1]' NARRATION2 = '[NARRATION2]' NARRATION3 = '[NARRATION3]' # The amount being posted. AMOUNT = '[AMOUNT]' # Debits and credits being posted in separate, dedicated columns. AMOUNT_DEBIT = '[DEBIT]' AMOUNT_CREDIT = '[CREDIT]' # The balance amount, after the row has posted. BALANCE = '[BALANCE]' # A field to use as a tag name. TAG = '[TAG]' # A field to use as a unique reference id or number. REFERENCE_ID = '[REF]' # A column which says DEBIT or CREDIT (generally ignored). DRCR = '[DRCR]' # Last 4 digits of the card. LAST4 = '[LAST4]' # An account name. ACCOUNT = '[ACCOUNT]' # Categorization, if the institution supports it. You could, in theory, # specialize your importer to use this automatically assign a good expenses # account. CATEGORY = '[CATEGORY]' # A column that indicates the amount currency for the current row which may # be different to the base currency. CURRENCY = '[CURRENCY]' def get_amounts(iconfig, row, allow_zero_amounts, parse_amount): """Get the amount columns of a row. Args: iconfig: A dict of Col to row index. row: A row array containing the values of the given row. allow_zero_amounts: Is a transaction with amount D('0.00') okay? If not, return (None, None). Returns: A pair of (debit-amount, credit-amount), both of which are either an instance of Decimal or None, or not available. """ debit, credit = None, None if Col.AMOUNT in iconfig: credit = row[iconfig[Col.AMOUNT]] else: debit = row[iconfig[Col.AMOUNT_DEBIT]] if Col.AMOUNT_DEBIT in iconfig else None credit = row[iconfig[Col.AMOUNT_CREDIT]] if Col.AMOUNT_CREDIT in iconfig else None # If zero amounts aren't allowed, return null value. is_zero_amount = ((credit is not None and parse_amount(credit) == ZERO) and (debit is not None and parse_amount(debit) == ZERO)) if not allow_zero_amounts and is_zero_amount: return (None, None) return (-parse_amount(debit) if debit else None, parse_amount(credit) if credit else None) def normalize_config(config, head, dialect='excel', skip_lines: int = 0): """Using the header line, convert the configuration field name lookups to int indexes. Args: config: A dict of Col types to string or indexes. head: A string, some decent number of bytes of the head of the file. dialect: A dialect definition to parse the header skip_lines: Skip first x (garbage) lines of file. Returns: A pair of A dict of Col types to integer indexes of the fields, and a boolean, true if the file has a header. Raises: ValueError: If there is no header and the configuration does not consist entirely of integer indexes. """ # Skip garbage lines before sniffing the header assert isinstance(skip_lines, int) assert skip_lines >= 0 head = io.StringIO(head, newline=None) lines = list(head)[skip_lines:] has_header = csv.Sniffer().has_header('\n'.join(lines)) if has_header: header = next(csv.reader(lines, dialect=dialect)) field_map = {name.strip(): index for index, name in enumerate(header)} index_config = {} for field_type, field in config.items(): if isinstance(field, str): field = field_map[field] index_config[field_type] = field else: if any(not isinstance(field, int) for field_type, field in config.items()): raise ValueError("CSV config without header has non-index fields: " "{}".format(config)) index_config = config return index_config, has_header def prepare_for_identifier(regexps: Union[str, List[str]], matchers: Optional[List[str]]) -> Dict[str, str]: """Prepare data for identifier mixin.""" if isinstance(regexps, str): regexps = [regexps] matchers = matchers or [] matchers.append(('mime', 'text/csv')) if regexps: for regexp in regexps: matchers.append(('content', regexp)) return {'matchers': matchers} def prepare_for_filing(account: str, institution: Optional[str], prefix: Optional[str]) -> Dict[str, str]: """Prepare kwds for filing mixin.""" kwds = {'filing': account} if institution: prefix = kwds.get('prefix', None) assert prefix is None kwds['prefix'] = institution return kwds class _CSVImporterBase: """Base class for CSV importer implementations. Note that many existing importers are based on this; be careful with modification of the attribute names and types. See concrete implementations below. """ # pylint: disable=too-many-instance-attributes FLAG = flags.FLAG_OKAY def __init__(self, config, account, currency, regexps=None, skip_lines: int = 0, last4_map: Optional[Dict] = None, categorizer: Optional[Callable] = None, institution: Optional[str] = None, debug: bool = False, csv_dialect: Union[str, csv.Dialect] = 'excel', dateutil_kwds: Optional[Dict] = None, narration_sep: str = '; ', encoding: Optional[str] = None, invert_sign: Optional[bool] = False, **kwds): """Constructor. Args: config: A dict of Col enum types to the names or indexes of the columns. account: An account string, the account to post this to. currency: A currency string, the currency of this account. regexps: A list of regular expression strings. skip_lines: Skip first x (garbage) lines of file. last4_map: A dict that maps last 4 digits of the card to a friendly string. categorizer: A callable with two arguments (transaction, row) that can attach the other posting (usually expenses) to a transaction with only single posting. institution: An optional name of an institution to rename the files to. debug: Whether or not to print debug information csv_dialect: A `csv` dialect given either as string or as instance or subclass of `csv.Dialect`. dateutil_kwds: An optional dict defining the dateutil parser kwargs. narration_sep: A string, a separator to use for splitting up the payee and narration fields of a source field. encoding: Encoding for the file, utf-8 if not specified or None. invert_sign: If true, invert the amount's sign unconditionally. **kwds: Extra keyword arguments to provide to the base mixins. """ assert isinstance(config, dict), "Invalid type: {}".format(config) self.config = config self.currency = currency assert isinstance(skip_lines, int) self.skip_lines = skip_lines self.last4_map = last4_map or {} self.debug = debug self.dateutil_kwds = dateutil_kwds self.csv_dialect = csv_dialect self.narration_sep = narration_sep self.encoding = encoding or 'utf-8' self.invert_sign = invert_sign self.categorizer = categorizer super().__init__(**kwds) def file_date(self, file): "Get the maximum date from the file." iconfig, has_header = normalize_config( self.config, file.head(encoding=self.encoding), self.csv_dialect, self.skip_lines, ) if Col.DATE in iconfig: with open(file.name, encoding=self.encoding) as infile: reader = iter(csv.reader(infile, dialect=self.csv_dialect)) for _ in range(self.skip_lines): next(reader) if has_header: next(reader) max_date = None for row in reader: if not row: continue if row[0].startswith('#'): continue date_str = row[iconfig[Col.DATE]] date = date_utils.parse_date(date_str, self.dateutil_kwds) if max_date is None or date > max_date: max_date = date return max_date def _do_extract(self, file, account, existing_entries=None): entries = [] # Normalize the configuration to fetch by index. iconfig, has_header = normalize_config( self.config, file.head(encoding=self.encoding), self.csv_dialect, self.skip_lines, ) with open(file.name, encoding=self.encoding) as infile: reader = iter(csv.reader(infile, dialect=self.csv_dialect)) # Skip garbage lines for _ in range(self.skip_lines): next(reader) # Skip header, if one was detected. if has_header: next(reader) def get(row, ftype): try: return row[iconfig[ftype]] if ftype in iconfig else None except IndexError: # FIXME: this should not happen return None # Parse all the transactions. first_row = last_row = None for index, row in enumerate(reader, 1): if not row: continue if row[0].startswith('#'): continue # If debugging, print out the rows. if self.debug: print(row) if first_row is None: first_row = row last_row = row # Extract the data we need from the row, based on the configuration. date = get(row, Col.DATE) txn_date = get(row, Col.TXN_DATE) txn_time = get(row, Col.TXN_TIME) payee = get(row, Col.PAYEE) if payee: payee = payee.strip() fields = filter(None, [get(row, field) for field in (Col.NARRATION1, Col.NARRATION2, Col.NARRATION3)]) narration = self.narration_sep.join( field.strip() for field in fields).replace('\n', '; ') tag = get(row, Col.TAG) tags = {tag} if tag else data.EMPTY_SET link = get(row, Col.REFERENCE_ID) links = {link} if link else data.EMPTY_SET last4 = get(row, Col.LAST4) balance = get(row, Col.BALANCE) currency = get(row, Col.CURRENCY) or self.currency # Create a transaction meta = data.new_metadata(file.name, index) if txn_date is not None: meta['date'] = date_utils.parse_date(txn_date, self.dateutil_kwds) if txn_time is not None: meta['time'] = str(dateutil.parser.parse(txn_time).time()) if balance is not None: meta['balance'] = Amount(self.parse_amount(balance), currency) if last4: last4_friendly = self.last4_map.get(last4.strip()) meta['card'] = last4_friendly if last4_friendly else last4 date = date_utils.parse_date(date, self.dateutil_kwds) txn = data.Transaction(meta, date, self.FLAG, payee, narration, tags, links, []) # Attach one posting to the transaction amount_debit, amount_credit = self.get_amounts(iconfig, row, False, self.parse_amount) # Skip empty transactions if amount_debit is None and amount_credit is None: continue for amount in [amount_debit, amount_credit]: if amount is None: continue if self.invert_sign: amount = -amount units = Amount(amount, currency) txn.postings.append( data.Posting(account, units, None, None, None, None)) # Attach the other posting(s) to the transaction. txn = self.call_categorizer(txn, row) # Add the transaction to the output list entries.append(txn) # Figure out if the file is in ascending or descending order. first_date = date_utils.parse_date(get(first_row, Col.DATE), self.dateutil_kwds) last_date = date_utils.parse_date(get(last_row, Col.DATE), self.dateutil_kwds) is_ascending = first_date < last_date # Reverse the list if the file is in descending order if not is_ascending: entries = list(reversed(entries)) # Add a balance entry if possible. If more than one currency # can appear in the input, add one balance statement for each. if Col.BALANCE in iconfig: balances = set() for entry in reversed(entries): # Remove the 'balance' metadata. balance = entry.meta.pop('balance', None) if balance is None: continue # Only add the newest entry for each currency in the file if balance.currency not in balances: date = entry.date + datetime.timedelta(days=1) meta = data.new_metadata(file.name, index) entries.append(data.Balance(meta, date, account, balance, None, None)) balances.add(balance.currency) return entries def call_categorizer(self, txn, row): if not isinstance(self.categorizer, collections.abc.Callable): return txn # TODO(blais): Remove introspection here, just commit to the two # parameter version. params = signature(self.categorizer).parameters if len(params) < 2: return self.categorizer(txn) return self.categorizer(txn, row) def parse_amount(self, string): """The method used to create Decimal instances. You can override this.""" return D(string) def get_amounts(self, iconfig, row, allow_zero_amounts, parse_amount): """See function get_amounts() for details. This method is present to allow clients to override it in order to deal with special cases, e.g., columns with currency symbols in them. """ return get_amounts(iconfig, row, allow_zero_amounts, parse_amount) # Deprecated. TODO(blais): Remove this eventually (on a major release). class Importer(_CSVImporterBase, identifier.IdentifyMixin, filing.FilingMixin): """Importer for CSV files. This class implements the old ImporterProtocol interface. It is deprecated and will be removed eventually. Use the CSVImporter class instead. """ def __init__(self, config, account, currency, regexps=None, skip_lines: int = 0, last4_map: Optional[Dict] = None, categorizer: Optional[Callable] = None, institution: Optional[str] = None, debug: bool = False, csv_dialect: Union[str, csv.Dialect] = 'excel', dateutil_kwds: Optional[Dict] = None, narration_sep: str = '; ', encoding: Optional[str] = None, invert_sign: Optional[bool] = False, **kwds): warnings.warn('beangulp.importers.csv.Importer is deprecated. ' 'Base your importer on beangulp.importers.csvbase.Importer instead.', DeprecationWarning, stacklevel=2) kwds.update(prepare_for_identifier(regexps, kwds.get('matchers'))) kwds.update(prepare_for_filing(account, kwds.get('prefix', None), institution)) super().__init__(config, account, currency, regexps=regexps, skip_lines=skip_lines, last4_map=last4_map, categorizer=categorizer, institution=institution, debug=debug, csv_dialect=csv_dialect, dateutil_kwds=dateutil_kwds, narration_sep=narration_sep, encoding=encoding, invert_sign=invert_sign, **kwds ) def extract(self, file, existing_entries=None): account = self.file_account(file) return self._do_extract(file, account, existing_entries) class CSVImporter(importer.Importer): """Importer for CSV files. This class adapts the old CSV code to implement the new redesigned Importer interface. The new beangulp.importers.csvbase.Importer may be a better base for newly developed importers. """ def __init__(self, config, account, currency, regexps=None, skip_lines: int = 0, last4_map: Optional[Dict] = None, categorizer: Optional[Callable] = None, institution: Optional[str] = None, debug: bool = False, csv_dialect: Union[str, csv.Dialect] = 'excel', dateutil_kwds: Optional[Dict] = None, narration_sep: str = '; ', encoding: Optional[str] = None, invert_sign: Optional[bool] = False, **kwds): """See _CSVImporterBase.""" self.base = _CSVImporterBase(config, account, currency, regexps, skip_lines, last4_map, categorizer, institution, debug, csv_dialect, dateutil_kwds, narration_sep, encoding, invert_sign) filing_kwds = prepare_for_filing(account, kwds.get('prefix', None), institution) self.filing = filing.FilingMixin(**filing_kwds) ident_kwds = prepare_for_identifier(regexps, kwds.get('matchers')) self.ident = identifier.IdentifyMixin(**ident_kwds) def identify(self, filepath): return self.ident.identify(cache.get_file(filepath)) def account(self, filepath): return self.filing.file_account(cache.get_file(filepath)) def date(self, filepath): return self.base.file_date(cache.get_file(filepath)) def filename(self, filepath): return path.basename(utils.idify(filepath)) def extract(self, filepath, existing=None): account = self.account(filepath) return self.base._do_extract(cache.get_file(filepath), account, existing) beangulp-0.2.0/beangulp/importers/csv_test.py000066400000000000000000000515621474340320100213420ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import tempfile import textwrap import unittest import warnings from pprint import pformat from beancount.core import data from beancount.parser import cmptest from beancount.utils import test_utils from beangulp.importers import csv Col = csv.Col class TestCSVFunctions(unittest.TestCase): def test_normalize_config__with_header(self): head = textwrap.dedent("""\ Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #, DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, """) iconfig, has_header = csv.normalize_config({Col.DATE: 'Posting Date'}, head) self.assertEqual({Col.DATE: 1}, iconfig) self.assertTrue(has_header) iconfig, _ = csv.normalize_config({Col.NARRATION: 'Description'}, head) self.assertEqual({Col.NARRATION: 2}, iconfig) iconfig, _ = csv.normalize_config({Col.DATE: 1, Col.NARRATION: 'Check or Slip #'}, head) self.assertEqual({Col.DATE: 1, Col.NARRATION: 6}, iconfig) iconfig, _ = csv.normalize_config({Col.DATE: 1}, head) self.assertEqual({Col.DATE: 1}, iconfig) def test_normalize_config__with_skip_and_header(self): head = textwrap.dedent("""\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam lorem erat, bibendum sed arcu at, tempor commodo tortor. Phasellus consectetur, nisl quis vestibulum ornare, mi velit imperdiet arcu, eu mattis nulla augue nec ex. Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #, DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, """) iconfig, has_header = csv.normalize_config({Col.DATE: 'Posting Date'}, head, skip_lines=4) self.assertEqual({Col.DATE: 1}, iconfig) self.assertTrue(has_header) iconfig, _ = csv.normalize_config({Col.NARRATION: 'Description'}, head, skip_lines=4) self.assertEqual({Col.NARRATION: 2}, iconfig) iconfig, _ = csv.normalize_config({Col.DATE: 1, Col.NARRATION: 'Check or Slip #'}, head, skip_lines=4) self.assertEqual({Col.DATE: 1, Col.NARRATION: 6}, iconfig) iconfig, _ = csv.normalize_config({Col.DATE: 1}, head, skip_lines=4) self.assertEqual({Col.DATE: 1}, iconfig) def test_normalize_config__without_header(self): head = textwrap.dedent("""\ DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, """) iconfig, has_header = csv.normalize_config({Col.DATE: 1}, head) self.assertEqual({Col.DATE: 1}, iconfig) self.assertFalse(has_header) with self.assertRaises(ValueError): csv.normalize_config({Col.DATE: 'Posting Date'}, head) def test_normalize_config__with_skip_and_without_header(self): head = textwrap.dedent("""\ Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aliquam lorem erat, bibendum sed arcu at, tempor commodo tortor. Phasellus consectetur, nisl quis vestibulum ornare, mi velit imperdiet arcu, eu mattis nulla augue nec ex. DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, """) iconfig, has_header = csv.normalize_config({Col.DATE: 1}, head, skip_lines=4) self.assertEqual({Col.DATE: 1}, iconfig) self.assertFalse(has_header) with self.assertRaises(ValueError): csv.normalize_config({Col.DATE: 'Posting Date'}, head, skip_lines=4) class TestCSVImporter(cmptest.TestCase): def setUp(self): warnings.simplefilter('ignore', category=DeprecationWarning) @test_utils.docfile def test_column_types(self, filename): # pylint: disable=line-too-long """\ Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #, DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, DEBIT,3/6/2016,"ATM WITHDRAWAL 001234 03/8888 DELANC",-60.00,ATM,3650.02,, CREDIT,3/5/2016,"CA STATE NYSTTAXRFD PPD ID: 1111111111",110.00,ACH_CREDIT,3710.02,, DEBIT,3/4/2016,"BOOGLE WALLET US000NEI9T WEB ID: C234567890",-1300.00,ACH_DEBIT,3600.02,, """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.NARRATION2: 'Check or Slip #', Col.AMOUNT: 'Amount', Col.BALANCE: 'Balance', Col.DRCR: 'Details'}, 'Assets:Bank', 'USD', ('Details,Posting Date,"Description",Amount,' 'Type,Balance,Check or Slip #,'), institution='chafe') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "Payment to Chafe card ending in 1234 03/18" Assets:Bank -2680.89 USD 2016-03-15 * "EMPLOYER INC DIRECT DEP PPD ID: 1111111111" Assets:Bank 2590.73 USD 2016-03-14 * "INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789" Assets:Bank -150.00 USD 2016-03-06 * "ATM WITHDRAWAL 001234 03/8888 DELANC" Assets:Bank -60.00 USD 2016-03-05 * "CA STATE NYSTTAXRFD PPD ID: 1111111111" Assets:Bank 110.00 USD 2016-03-04 * "BOOGLE WALLET US000NEI9T WEB ID: C234567890" Assets:Bank -1300.00 USD 2016-03-19 balance Assets:Bank 3409.86 USD """, entries) @test_utils.docfile def test_date_formats(self, filename): """\ Posting,Description,Amount 11/7/2016,A,2 12/7/2016,B,3 13/7/2016,C,4 """ importer = csv.CSVImporter({Col.DATE: 'Posting', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount'}, 'Assets:Bank', 'EUR', [], dateutil_kwds={'dayfirst': True}) entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-07-11 * "A" Assets:Bank 2 EUR 2016-07-12 * "B" Assets:Bank 3 EUR 2016-07-13 * "C" Assets:Bank 4 EUR """, entries) @test_utils.docfile def test_links(self, filename): """\ Date,Description,Amount,Link 2020-07-03,A,2, 2020-07-03,B,3,123 """ importer = csv.CSVImporter({Col.DATE: 'Date', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount', Col.REFERENCE_ID: 'Link'}, 'Assets:Bank', 'EUR', []) entries = importer.extract(filename) self.assertEqualEntries(r""" 2020-07-03 * "A" Assets:Bank 2 EUR 2020-07-03 * "B" ^123 Assets:Bank 3 EUR """, entries) @test_utils.docfile def test_tags(self, filename): """\ Date,Description,Amount,Tag 2020-07-03,A,2, 2020-07-03,B,3,foo """ importer = csv.CSVImporter({Col.DATE: 'Date', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount', Col.TAG: 'Tag'}, 'Assets:Bank', 'EUR', []) entries = importer.extract(filename) self.assertEqualEntries(r""" 2020-07-03 * "A" Assets:Bank 2 EUR 2020-07-03 * "B" #foo Assets:Bank 3 EUR """, entries) @test_utils.docfile def test_zero_balance_produces_assertion(self, filename): """\ Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #, DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,0,, """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.NARRATION2: 'Check or Slip #', Col.AMOUNT: 'Amount', Col.BALANCE: 'Balance', Col.DRCR: 'Details'}, 'Assets:Bank', 'USD', ('Details,Posting Date,"Description",Amount,' 'Type,Balance,Check or Slip #,'), institution='chafe') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "Payment to Chafe card ending in 1234 03/18" Assets:Bank -2680.89 USD 2016-03-19 balance Assets:Bank 0 USD """, entries) @test_utils.docfile def test_currency_and_balances_where_there_are_multiple_currency_transactions(self, filename): """\ Posting Date,"Description",Amount,Currency,Balance 3/18/2016,"1st Payment in GBP",-1.00,GBP,-1 3/18/2016,"1st Payment in PLN",-1,PLN,-1 3/18/2016,"1st Payment in ZAR",-1.0,ZAR,-1 3/19/2016,"2nd Payment in GBP",-2,GBP,-3 3/19/2016,"2nd Payment in Main Currency",-2.00,,-3 3/20/2016,"3rd Payment in GBP",-3,GBP,-6 """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.AMOUNT: 'Amount', Col.CURRENCY: 'Currency', Col.BALANCE: 'Balance'}, 'Assets:Bank', 'PLN') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "1st Payment in GBP" Assets:Bank -1.00 GBP 2016-03-18 * "1st Payment in PLN" Assets:Bank -1 PLN 2016-03-18 * "1st Payment in ZAR" Assets:Bank -1.0 ZAR 2016-03-19 * "2nd Payment in GBP" Assets:Bank -2 GBP 2016-03-19 * "2nd Payment in Main Currency" Assets:Bank -2.00 PLN 2016-03-20 * "3rd Payment in GBP" Assets:Bank -3 GBP 2016-03-20 balance Assets:Bank -3 PLN 2016-03-19 balance Assets:Bank -1 ZAR 2016-03-21 balance Assets:Bank -6 GBP """, entries) @test_utils.docfile def test_zero_balance_assertion_is_added_with_currency_field(self, filename): """\ Posting Date,"Description",Amount,Currency,Balance 3/18/2016,"1st Payment in GBP",-1.00,GBP,-1 3/18/2016,"1st Payment in PLN",-1,PLN,-1 3/18/2016,"1st Payment in ZAR",-1.0,ZAR,-1 3/19/2016,"2nd Payment in GBP",-2,GBP,-3 3/19/2016,"2nd Payment in Main Currency",-2.00,,-3 3/20/2016,"3rd Payment in GBP",-3,GBP,-6 3/21/2016,"4th Payment in GBP",6,GBP,0 """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.AMOUNT: 'Amount', Col.CURRENCY: 'Currency', Col.BALANCE: 'Balance'}, 'Assets:Bank', 'PLN') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "1st Payment in GBP" Assets:Bank -1.00 GBP 2016-03-18 * "1st Payment in PLN" Assets:Bank -1 PLN 2016-03-18 * "1st Payment in ZAR" Assets:Bank -1.0 ZAR 2016-03-19 * "2nd Payment in GBP" Assets:Bank -2 GBP 2016-03-19 * "2nd Payment in Main Currency" Assets:Bank -2.00 PLN 2016-03-20 * "3rd Payment in GBP" Assets:Bank -3 GBP 2016-03-21 * "4th Payment in GBP" Assets:Bank 6 GBP 2016-03-20 balance Assets:Bank -3 PLN 2016-03-19 balance Assets:Bank -1 ZAR 2016-03-22 balance Assets:Bank 0 GBP """, entries) @test_utils.docfile def test_currency_and_balances_when_none_are_in_the_main_currency(self, filename): """\ Posting Date,"Description",Amount,Currency,Balance 3/18/2016,"1st Payment in GBP",-1.00,GBP,-1 3/18/2016,"1st Payment in ZAR",-1.0,ZAR,-1 3/19/2016,"2nd Payment in GBP",-2,GBP,-3 3/20/2016,"3rd Payment in GBP",-3,GBP,-6 """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.AMOUNT: 'Amount', Col.CURRENCY: 'Currency', Col.BALANCE: 'Balance'}, 'Assets:Bank', 'PLN') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "1st Payment in GBP" Assets:Bank -1.00 GBP 2016-03-18 * "1st Payment in ZAR" Assets:Bank -1.0 ZAR 2016-03-19 * "2nd Payment in GBP" Assets:Bank -2 GBP 2016-03-20 * "3rd Payment in GBP" Assets:Bank -3 GBP 2016-03-19 balance Assets:Bank -1 ZAR 2016-03-21 balance Assets:Bank -6 GBP """, entries) @test_utils.docfile def test_main_currency_should_be_used_when_no_currency_is_specified(self, filename): """\ Posting Date,"Description",Amount,Currency,Balance 3/18/2016,"1st Payment",-1.00,,-1 3/18/2016,"2nd Payment",-1.0,,-2 3/19/2016,"3rd Payment",-2,,-4 3/20/2016,"4th Payment",-3,,-7 """ importer = csv.CSVImporter({Col.DATE: 'Posting Date', Col.NARRATION1: 'Description', Col.AMOUNT: 'Amount', Col.CURRENCY: 'Currency', Col.BALANCE: 'Balance'}, 'Assets:Bank', 'PLN') entries = importer.extract(filename) self.assertEqualEntries(r""" 2016-03-18 * "1st Payment" Assets:Bank -1.00 PLN 2016-03-18 * "2nd Payment" Assets:Bank -1.0 PLN 2016-03-19 * "3rd Payment" Assets:Bank -2 PLN 2016-03-20 * "4th Payment" Assets:Bank -3 PLN 2016-03-21 balance Assets:Bank -7 PLN """, entries) @test_utils.docfile def test_categorizer_one_argument(self, filename): """\ Date,Amount,Payee,Description 6/2/2020,30.00,"Payee here","Description" 7/2/2020,-25.00,"Supermarket","Groceries" """ def categorizer(txn): if txn.narration == "Groceries": txn.postings.append( data.Posting("Expenses:Groceries", -txn.postings[0].units, None, None, None, None)) return txn importer = csv.CSVImporter({Col.DATE: 'Date', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount'}, 'Assets:Bank', 'EUR', ('Date,Amount,Payee,Description'), categorizer=categorizer, institution='foobar') entries = importer.extract(filename) self.assertEqualEntries(r""" 2020-06-02 * "Description" Assets:Bank 30.00 EUR 2020-07-02 * "Groceries" Assets:Bank -25.00 EUR Expenses:Groceries 25.00 EUR """, entries) @test_utils.docfile def test_categorizer_two_arguments(self, filename): """\ Date,Amount,Payee,Description 6/2/2020,30.00,"Payee here","Description" 7/2/2020,-25.00,"Supermarket","Groceries" """ def categorizer(txn, row): txn = txn._replace(payee=row[2]) txn.meta['source'] = pformat(row) return txn importer = csv.CSVImporter({Col.DATE: 'Date', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount'}, 'Assets:Bank', 'EUR', ('Date,Amount,Payee,Description'), categorizer=categorizer, institution='foobar') entries = importer.extract(filename) self.assertEqualEntries(r""" 2020-06-02 * "Payee here" "Description" source: "['6/2/2020', '30.00', 'Supermarket', 'Groceries']" Assets:Bank 30.00 EUR 2020-07-02 * "Supermarket" "Groceries" source: "['7/2/2020', '-25.00', 'Supermarket', 'Groceries']" Assets:Bank -25.00 EUR """, entries) @test_utils.docfile def test_explict_encoding_utf8(self, filename): """\ Posting,Description,Amount 2020/08/08,🍏,2 """ importer = csv.CSVImporter({Col.DATE: 'Posting', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount'}, 'Assets:Bank', 'EUR', [], encoding='utf-8') entries = importer.extract(filename) self.assertEqualEntries(r""" 2020-08-08 * "🍏" Assets:Bank 2 EUR """, entries) def test_newlines(self): content = textwrap.dedent("""\ Date,Description,Amount 2020-07-03,A,2 2020-07-03,B,3 """) importer = csv.CSVImporter({Col.DATE: 'Date', Col.NARRATION: 'Description', Col.AMOUNT: 'Amount'}, 'Assets:Bank', 'EUR', []) for nl in '\n', '\r\n', '\r': with tempfile.NamedTemporaryFile('w') as temp: temp.write(content.replace('\n', nl)) temp.flush() entries = importer.extract(temp.name) self.assertEqualEntries(""" 2020-07-03 * "A" Assets:Bank 2 EUR 2020-07-03 * "B" Assets:Bank 3 EUR """, entries) # TODO: Test things out with/without payee and with/without narration. # TODO: Test balance support. # TODO: Add balances every month or week. # TODO: Test ascending and descending orders. # TODO: Add a test for all the supported fields, e.g. NARRATION2. beangulp-0.2.0/beangulp/importers/csvbase.py000066400000000000000000000300151474340320100211240ustar00rootroot00000000000000import abc import csv import datetime import decimal from enum import Enum import re from collections import defaultdict from itertools import islice from beancount.core import data import beangulp EMPTY = frozenset() def _resolve(spec, names): """Resolve column specification into column index. Args: spec: Column name or index. names: A dict mapping column names to column indices. Returns: Column index. """ if isinstance(spec, int): return spec if names is None: raise KeyError(f'Column {spec!r} cannot be found in file without column names') col = names.get(spec) if col is None: cols = ', '.join(repr(name) for name in names.keys()) raise KeyError(f'Cannot find column {spec!r} in column names: {cols}') return col class Column: """Field descriptor. Args: name: Column name or index. default: Value to return if the field is empty. """ def __init__(self, *names, default=None): self.names = names self.default = default def __repr__(self): names = ', '.join(repr(i) for i in self.names) return f'{self.__class__.__module__}.{self.__class__.__name__}({names})' def getter(self, names): """Generate an attribute accessor for the column specification. The returned function is a getitem-like function that takes a tuple as argument and returns the value of the field described by the column specificaion. The function returns the field default value if all columns are empty. The value is parsed with the column parser function. Args: names: A dict mapping column names to column indices. Returns: An accessor function. """ idxs = [_resolve(x, names) for x in self.names] def func(obj): value = tuple(obj[i] for i in idxs) if not all(value) and self.default: return self.default return self.parse(*value) return func def parse(self, value): """Parse column value. Args: value: Field string value obtained from the CSV reader. Returns: Field parsed value. """ return value.strip() class Columns(Column): """Specialized Column for multiple column fields. Args: name: Column names or indexes. sep: Separator to use to join columns. """ def __init__(self, *names, sep=' '): super().__init__(*names) self.sep = sep def parse(self, *values): return self.sep.join(val.strip() for val in values if val) class Date(Column): """Specialized Column descriptor for date fields. Parse strings into datetime.date objects accordingly to the provided format specification. The format specification is the same understood by datetime.datetime.strptime(). Args: name: Column name or index. frmt: Date format specification. """ def __init__(self, name, frmt='%Y-%m-%d'): super().__init__(name) self.frmt = frmt def parse(self, value): return datetime.datetime.strptime(value.strip(), self.frmt).date() class Amount(Column): """Specialized Column descriptor for decimal fields. Parse strings into decimal.Decimal objects. Optionally apply regexp substitutions before parsing the decimal number. This allows to normalize locale formatted decimal numbers into the format expected by decimal.Decimal(). Args: name: Column name or index. subs: Dictionary mapping regular expression patterns to replacement strings. Substitutions are performed with re.sub() in the order they are specified. """ def __init__(self, name, subs=None): super().__init__(name) self.subs = subs if subs is not None else {} def parse(self, value): for pattern, replacement in self.subs.items(): value = re.sub(pattern, replacement, value) return decimal.Decimal(value) # The CSV Importer class needs to inherit from beangulp.Importer which # is an abstract base class having abc.ABCMeta as metaclass. To be # able to do so out CSV metaclass need to be a sublcass of # abc.ABCMeta. class CSVMeta(abc.ABCMeta): """A metaclass that extracts column specifications from class members and stores them in a columns dictionary keyed by the member name.""" def __new__(mcs, name, bases, dct): columns = {} others = {} for key, value in dct.items(): if isinstance(value, Column): columns[key] = value continue others[key] = value others['columns'] = columns return super().__new__(mcs, name, bases, others) class Order(Enum): ASCENDING = 1 """Entries are listed in chronological order.""" DESCENDING = 2 """Entries are listed in reverse chronological order.""" class CSVReader(metaclass=CSVMeta): encoding = 'utf8' """File encoding.""" skiplines = 0 """Number of input lines to skip before starting processing.""" names = True """Whether the data file contains a row with column names.""" dialect = None """The CSV dialect used in the input file.""" comments = '#' """Comment character.""" order = None """Order of entries in the CSV file. If None the order will be inferred from the file content.""" # This is populated by the CSVMeta metaclass. columns = {} def read(self, filepath): """Read CSV file according to class defined columns specification. Use the first rown in the data file to resolve columns specification. Yield namedtuple-like objects with attribute accessors to access the data fields as defined by the class columns specification. Args: filepath: Filesystem path to the input file. Yields: Namedtuple-like objects. """ with open(filepath, encoding=self.encoding) as fd: # Skip header lines. lines = islice(fd, self.skiplines, None) # Filter out comment lines. if self.comments: lines = filter(lambda x: not x.startswith(self.comments), lines) reader = csv.reader(lines, dialect=self.dialect) # Map column names to column indices. names = None if self.names: headers = next(reader, None) if headers is None: raise IndexError('The input file does not contain an header line') names = {name.strip(): index for index, name in enumerate(headers)} # Construct a class with attribute accessors for the # configured columns that works similarly to a namedtuple. attrs = {} for name, column in self.columns.items(): attrs[name] = property(column.getter(names)) row = type('Row', (tuple, ), attrs) # Return data rows. for x in reader: yield row(x) class Importer(beangulp.Importer, CSVReader): """CSV files importer base class. This class provides the infrastructure and the basic functionality necessary for importing transactions and balance assertions from CSV files. To do anything useful it needs to be subclassed to add fields definitions. Fields are defined as class attributes of type Column. Args: account: Importer default account. currency: Importer default currency. flag: Importer default flag for new transactions. """ def __init__(self, account, currency, flag='*'): self.importer_account = account self.currency = currency self.flag = flag def date(self, filepath): """Implement beangulp.Importer::date()""" return max(row.date for row in self.read(filepath)) def account(self, filepath): """Implement beangulp.Importer::account()""" return self.importer_account def extract(self, filepath, existing): """Implement beangulp.Importer::extract() This methods costructs a transaction for each data row using the date, narration, and amount required fields and the flag, payee, account, currency, tag, link, balance optional fields. Transaction metadata is constructed with the metadata() method and the finalize() method is called on each transaction. These can be redefine in subclasses. For customization that cannot be implemented with these two extension points, consider basing the importer on the CSVReader class and implement tailored data processing in the extract() method. """ entries = [] balances = defaultdict(list) default_account = self.account(filepath) # Compute the line number of the first data line. offset = int(self.skiplines) + bool(self.names) + 1 for lineno, row in enumerate(self.read(filepath), offset): # Skip empty lines. if not row: continue tag = getattr(row, 'tag', None) tags = {tag} if tag else EMPTY link = getattr(row, 'link', None) links = {link} if link else EMPTY # This looks like an exercise in defensive programming # gone too far, but we do not want to depend on any field # being defined other than the essential ones. flag = getattr(row, 'flag', self.flag) payee = getattr(row, 'payee', None) account = getattr(row, 'account', default_account) currency = getattr(row, 'currency', self.currency) units = data.Amount(row.amount, currency) # Create a transaction. txn = data.Transaction(self.metadata(filepath, lineno, row), row.date, flag, payee, row.narration, tags, links, [ data.Posting(account, units, None, None, None, None), ]) # Apply user processing to the transaction. txn = self.finalize(txn, row) if txn is None: continue # Add the transaction to the output list. entries.append(txn) # Add balance to balances list. balance = getattr(row, 'balance', None) if balance is not None: date = row.date + datetime.timedelta(days=1) units = data.Amount(balance, currency) meta = data.new_metadata(filepath, lineno) balances[currency].append(data.Balance(meta, date, account, units, None, None)) if not entries: return [] if self.order is None: self.order = Order.ASCENDING if entries[0].date <= entries[-1].date else Order.DESCENDING # Reverse the list if the file is in descending order. if self.order is Order.DESCENDING: entries.reverse() # Append balances. for currency, balances in balances.items(): entries.append(balances[-1 if self.order is Order.ASCENDING else 0]) return entries def metadata(self, filepath, lineno, row): """Build transaction metadata dictionary. This method can be extended to add customized metadata entries based on the content of the data row. Args: filepath: Path to the file being imported. lineno: Line number of the data being processed. row: The data row being processed. Returns: A metadata dictionary. """ return data.new_metadata(filepath, lineno) def finalize(self, txn, row): """Post process the transaction. Returning None results in the transaction being discarded and in source row to do not contribute to the determination of the balances. Args: txn: The just build Transaction object. row: The data row being processed. Returns: A potentially extended or modified Transaction object or None. """ return txn beangulp-0.2.0/beangulp/importers/csvbase_test.py000066400000000000000000000401051474340320100221640ustar00rootroot00000000000000import datetime import decimal import unittest from beancount.core import data from beancount.parser import cmptest from beancount.utils.test_utils import docfile from beangulp.importers.csvbase import Column, Columns, Date, Amount, CSVMeta, CSVReader, Order, Importer class TestColumn(unittest.TestCase): def test_index_spec(self): column = Column(0) func = column.getter(None) value = func(('0', '1', '2', '3', )) self.assertEqual(value, '0') def test_named_spec(self): column = Column('Num') func = column.getter({'Num': 1}) value = func(('0', '1', '2', '3', )) self.assertEqual(value, '1') def test_named_errrors(self): column = Column('A') with self.assertRaisesRegex(KeyError, "Cannot find column 'A' in "): column.getter({'B': 0, 'C': 1}) with self.assertRaisesRegex(KeyError, "Column 'A' cannot be found in "): column.getter(None) def test_strip(self): # The default field parser strips whitespace. column = Column(0) func = column.getter(None) value = func((' value ', )) self.assertEqual(value, 'value') def test_default(self): column = Column(0, default=42) func = column.getter(None) value = func(('', )) self.assertEqual(value, 42) class TestDateColumn(unittest.TestCase): def test_default_format(self): column = Date(0) func = column.getter(None) value = func(('2021-05-16', )) self.assertEqual(value, datetime.date(2021, 5, 16)) def test_custom_format(self): column = Date(0, '%d.%m.%Y') func = column.getter(None) value = func(('16.05.2021', )) self.assertEqual(value, datetime.date(2021, 5, 16)) class TestColumnsColumn(unittest.TestCase): def test_default_sep(self): column = Columns(0, 1) func = column.getter(None) value = func(('0', '1', '2', '3', )) self.assertEqual(value, '0 1') def test_custom_sep(self): column = Columns(0, 1, sep=': ') func = column.getter(None) value = func(('0', '1', '2', '3', )) self.assertEqual(value, '0: 1') class TestAmountColumn(unittest.TestCase): def test_parse_decimal(self): column = Amount(0) func = column.getter(None) value = func(('1.0', )) self.assertIsInstance(value, decimal.Decimal) self.assertEqual(value, decimal.Decimal('1.0')) def test_parse_subs_one(self): column = Amount(0, subs={',': ''}) func = column.getter(None) value = func(('1,000.00', )) self.assertIsInstance(value, decimal.Decimal) self.assertEqual(value, decimal.Decimal('1000.00')) def test_parse_subs_two(self): column = Amount(0, subs={'\\.': '', ',': '.'}) func = column.getter(None) value = func(('1.000,00', )) self.assertIsInstance(value, decimal.Decimal) self.assertEqual(value, decimal.Decimal('1000.00')) def test_parse_subs_currency(self): column = Amount(0, subs={'\\$(.*)': '\\1', ',': ''}) func = column.getter(None) value = func(('$1,000.00', )) self.assertIsInstance(value, decimal.Decimal) self.assertEqual(value, decimal.Decimal('1000.00')) class TestCSVMeta(unittest.TestCase): def test_collect_fields(self): class CSVTest(metaclass=CSVMeta): first = Column(0) second = Column(1) # pylint: disable=no-member self.assertEqual(CSVTest.columns.keys(), {'first', 'second'}) class TestCSVReader(unittest.TestCase): @docfile def test_named_columns(self, filename): """\ First, Second 1, 2 3, 4 """ class Reader(CSVReader): first = Column('First') second = Column('Second') reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 2) # tuple elements self.assertEqual(rows[0][0], '1') # leading space is preserved self.assertEqual(rows[1][1], ' 4') # attribute accessors self.assertEqual(rows[0].first, '1') self.assertEqual(rows[1].second, '4') @docfile def test_named_no_enough_lines(self, filename): """\ # comment """ class Reader(CSVReader): first = Column('First') second = Column('Second') reader = Reader() with self.assertRaisesRegex(IndexError, 'The input file does not contain '): list(reader.read(filename)) @docfile def test_indexed_columns_names_false(self, filename): """\ First, Second 1, 2 3, 4 """ class Reader(CSVReader): first = Column(0) second = Column(1) names = False reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 3) # tuple elements self.assertEqual(rows[0][0], 'First') # leading space is preserved self.assertEqual(rows[2][1], ' 4') # attribute accessors self.assertEqual(rows[0].first, 'First') self.assertEqual(rows[2].second, '4') @docfile def test_indexed_columns_names_true(self, filename): """\ First, Second 1, 2 3, 4 """ class Reader(CSVReader): first = Column(0) second = Column(1) reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 2) # tuple elements self.assertEqual(rows[0][0], '1') # leading space is preserved self.assertEqual(rows[1][1], ' 4') # attribute accessors self.assertEqual(rows[0].first, '1') self.assertEqual(rows[1].second, '4') @docfile def test_comments(self, filename): """\ First, Second # ignore a, b c, d """ class Reader(CSVReader): first = Column(0) second = Column(1) reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 2) self.assertEqual(rows[0][0], 'a') @docfile def test_no_comments(self, filename): """\ First, Second # ignore a, b c, d """ class Reader(CSVReader): first = Column(0) second = Column(1) comments = False reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 3) self.assertEqual(rows[0][0], '# ignore') @docfile def test_custom_comments(self, filename): """\ First, Second # ignore ; ignore a, b c, d """ class Reader(CSVReader): first = Column(0) second = Column(1) comments = ';' reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 3) self.assertEqual(rows[0][0], '# ignore') @docfile def test_skiplines(self, filename): """\ Skip Skip First, Second a, b """ class Reader(CSVReader): first = Column(0) second = Column(1) skiplines = 2 reader = Reader() rows = list(reader.read(filename)) self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], 'a') class Base(Importer): def identify(self, filepath): return True class TestImporter(cmptest.TestCase): @docfile def test_date(self, filename): """\ 2021-05-17, Test, 1.00 2021-05-18, Test, 1.00 2021-05-16, Test, 1.00 """ class CSVImporter(Base): date = Date(0) importer = CSVImporter('Account:CSV', 'EUR') date = importer.date(filename) self.assertEqual(date, datetime.date(2021, 5, 18)) @docfile def test_extract(self, filename): """\ 2021-05-17, Test, 1.00 """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test" Assets:CSV 1.00 EUR """) @docfile def test_extract_payee(self, filename): """\ 2021-05-17, Payee, Test, 1.00 """ class CSVImporter(Base): date = Date(0) payee = Column(1) narration = Column(2) amount = Amount(3) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Payee" "Test" Assets:CSV 1.00 EUR """) @docfile def test_extract_account(self, filename): """\ 2021-05-17, Test, Assets:Test, 1.00 """ class CSVImporter(Base): date = Date(0) narration = Column(1) account = Column(2) amount = Amount(3) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test" Assets:Test 1.00 EUR """) @docfile def test_extract_balance(self, filename): """\ 2021-05-18, Test A, 2.00, 3.00 2021-05-17, Test B, 1.00, 1.00 """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) balance = Amount(3) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test B" Assets:CSV 1.00 EUR 2021-05-18 * "Test A" Assets:CSV 2.00 EUR 2021-05-19 balance Assets:CSV 3.00 EUR """) @docfile def test_extract_balance_with_multiple_transactions_per_day_and_ascending_dates(self, filename): """\ 2021-05-17, Test A, 1.00, 1.00 2021-05-18, Test B, 2.00, 3.00 2021-05-18, Test C, 1.00, 4.00 """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) balance = Amount(3) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test A" Assets:CSV 1.00 EUR 2021-05-18 * "Test B" Assets:CSV 2.00 EUR 2021-05-18 * "Test C" Assets:CSV 1.00 EUR 2021-05-19 balance Assets:CSV 4.00 EUR """) @docfile def test_extract_balance_with_order_override(self, filename): """\ 2021-05-18, Test B, 2.00, 4.00 2021-05-18, Test C, 1.00, 2.00 """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) balance = Amount(3) names = False order = Order.DESCENDING importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-18 * "Test C" Assets:CSV 1.00 EUR 2021-05-18 * "Test B" Assets:CSV 2.00 EUR 2021-05-19 balance Assets:CSV 4.00 EUR """) @docfile def test_extract_flag(self, filename): """\ 2021-05-17, Test, 1.00, ! """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) flag = Column(3) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 ! "Test" Assets:CSV 1.00 EUR """) @docfile def test_extract_link_and_tag(self, filename): """\ 2021-05-17, Test, 1.00, link, tag """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) link = Column(3) tag = Column(4) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test" ^link #tag Assets:CSV 1.00 EUR """) @docfile def test_extract_currency(self, filename): """\ 2021-05-17, Test A US, 1.00, 1.00, USD 2021-05-17, Test A EU, 2.00, 2.00, EUR 2021-05-18, Test B US, 2.00, 3.00, USD 2021-05-18, Test B EU, 3.00, 5.00, EUR """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) balance = Amount(3) currency = Column(4) names = False importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test A US" Assets:CSV 1.00 USD 2021-05-17 * "Test A EU" Assets:CSV 2.00 EUR 2021-05-18 * "Test B US" Assets:CSV 2.00 USD 2021-05-18 * "Test B EU" Assets:CSV 3.00 EUR 2021-05-19 balance Assets:CSV 3.00 USD 2021-05-19 balance Assets:CSV 5.00 EUR """) @docfile def test_extract_metadata(self, filename): """\ 2021-05-17, Test, 1.00, data, 42 """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) meta = Column(3) data = Amount(4) names = False def metadata(self, filepath, lineno, row): meta = super().metadata(filepath, lineno, row) for field in 'meta', 'data': meta[field] = getattr(row, field) return meta importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test" meta: "data" data: 42 Assets:CSV 1.00 EUR """) @docfile def test_extract_finalize(self, filename): """\ 2021-05-17, Test, -1.00, Testing """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) category = Column(3) names = False def finalize(self, txn, row): posting = data.Posting( 'Expenses:' + row.category, # This could be None in a real importer. However, # the trsting framework accepts only complete # transactions, thus we do the booking manually. -txn.postings[0].units, None, None, None, None) txn.postings.append(posting) return txn importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqualEntries(entries, """ 2021-05-17 * "Test" Assets:CSV -1.00 EUR Expenses:Testing 1.00 EUR """) @docfile def test_extract_finalize_remove(self, filename): """\ 2021-05-17, Test, -1.00, Testing 2021-05-18, Drop, -2.00, Testing """ class CSVImporter(Base): date = Date(0) narration = Column(1) amount = Amount(2) names = False def finalize(self, txn, row): if txn.narration == 'Drop': return None return txn importer = CSVImporter('Assets:CSV', 'EUR') entries = importer.extract(filename, []) self.assertEqual(len(entries), 1) beangulp-0.2.0/beangulp/importers/fileonly.py000066400000000000000000000006771474340320100213320ustar00rootroot00000000000000"""A simplistic importer that can be used just to file away some download. Sometimes you just want to save and accumulate data """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" from beangulp.importers.mixins import filing from beangulp.importers.mixins import identifier class Importer(filing.FilingMixin, identifier.IdentifyMixin): """An importer that supports only matching (identification) and filing.""" beangulp-0.2.0/beangulp/importers/fileonly_test.py000066400000000000000000000024011474340320100223540ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import unittest from beancount.utils import test_utils from beangulp.importers import fileonly from beangulp import cache from beangulp import file_type class TestFileOnly(unittest.TestCase): def test_constructors(self): fileonly.Importer(matchers=[('filename', '.csv'), ('mime', 'text/plain')], filing='Assets:BofA:Checking', prefix='bofa') @unittest.skipIf(not file_type.magic, 'python-magic is not installed') @test_utils.docfile def test_match(self, filename): """\ DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-04-14,BUY,14167001,BOUGHT +CSKO 50 @98.35,7.95,-4925.45,25674.63 2014-05-08,BUY,12040838,BOUGHT +HOOL 121 @79.11,7.95,-9580.26,16094.37 """ importer = fileonly.Importer( matchers=[('filename', 'te?mp'), ('content', 'DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT')], filing='Assets:BofA:Checking', prefix='bofa') file = cache._FileMemo(filename) self.assertTrue(importer.identify(file)) assert importer.file_name(file).startswith('bofa.') beangulp-0.2.0/beangulp/importers/mixins/000077500000000000000000000000001474340320100204345ustar00rootroot00000000000000beangulp-0.2.0/beangulp/importers/mixins/__init__.py000066400000000000000000000000001474340320100225330ustar00rootroot00000000000000beangulp-0.2.0/beangulp/importers/mixins/config.py000066400000000000000000000035471474340320100222640ustar00rootroot00000000000000"""Base class that implements configuration and a filing account. """ __author__ = "Martin Blais " from beangulp import importer def validate_config(config, schema, importer): """Check the configuration account provided by the user against the accounts required by the source importer. Args: config: A config dict of actual values on an importer. schema: A dict of declarations of required values. Raises: ValueError: If the configuration is invalid. Returns: A validated configuration dict. """ provided_options = set(config) required_options = set(schema) for option in (required_options - provided_options): raise ValueError("Missing value from user configuration for importer {}: {}".format( importer.__class__.__name__, option)) for option in (provided_options - required_options): raise ValueError("Unknown value in user configuration for importer {}: {}".format( importer.__class__.__name__, option)) # FIXME: Validate types as well, including account type as a default. # FIXME: Here we could validate account names by looking them up from the # existing ledger. return config # FIXME: Add functionality to get account names with substitutions coming from # context, making it easier to have symbols in account names. class ConfigMixin(importer.ImporterProtocol): # Override this with the configuration. REQUIRED_CONFIG = None def __init__(self, **kwds): """Pull 'config' from kwds.""" config = kwds.pop('config', None) schema = self.REQUIRED_CONFIG if config or schema: assert config is not None assert schema is not None self.config = validate_config(config, config, self) else: self.config = None super().__init__(**kwds) beangulp-0.2.0/beangulp/importers/mixins/filing.py000066400000000000000000000027171474340320100222650ustar00rootroot00000000000000# TODO(blais): If we don't remove this file it should get renamed to archive.py. """Base class that implements filing account. It also sports an optional prefix to prepend to the renamed filename. Typically you can put the name of the institution there, so you get a renamed filename like this: YYYY-MM-DD.institution.Original_File_Name.pdf """ __author__ = "Martin Blais " from os import path from beancount.core import account from beangulp import importer class FilingMixin(importer.ImporterProtocol): def __init__(self, **kwds): """Pull 'filing' and 'prefix' from kwds. Args: filing: The name of the account to file to. prefix: The name of the institution prefix to insert. """ self.filing_account = kwds.pop('filing', None) assert account.is_valid(self.filing_account) self.prefix = kwds.pop('prefix', None) super().__init__(**kwds) def name(self): """Include the filing account in the name.""" return '{}: "{}"'.format(super().name(), self.filing_account) def file_account(self, file): return self.filing_account def file_name(self, file): """Return the optional renamed account filename.""" supername = super().file_name(file) if not self.prefix: return supername return '.'.join(filter(None, [self.prefix, supername or path.basename(file.name)])) beangulp-0.2.0/beangulp/importers/mixins/identifier.py000066400000000000000000000042631474340320100231350ustar00rootroot00000000000000"""Base class that implements identification using regular expressions. """ __author__ = "Martin Blais " import collections import itertools import re from beangulp import cache from beangulp import importer _PARTS = {'mime', 'filename', 'content'} def identify(remap, converter, file): """Identify the contents of a file. Args: remap: A dict of 'part' to list-of-compiled-regexp objects, where each item is a specification to match against its part. The 'part' can be one of 'mime', 'filename' or 'content'. converter: A function to convert the given file. Returns: A boolean, true if the file is not rejected by the constraints. """ if remap.get('mime', None): mimetype = file.convert(cache.mimetype) if mimetype is None: return False if not all(regexp.search(mimetype) for regexp in remap['mime']): return False if remap.get('filename', None): if not all(regexp.search(file.name) for regexp in remap['filename']): return False if remap.get('content', None): # If this is a text file, read the whole thing in memory. text = file.convert(converter or cache.contents) if not all(regexp.search(text) for regexp in remap['content']): return False return True class IdentifyMixin(importer.ImporterProtocol): def __init__(self, **kwds): """Pull 'matchers' and 'converter' from kwds.""" self.remap = collections.defaultdict(list) matchers = kwds.pop('matchers', []) cls_matchers = getattr(self, 'matchers', []) assert isinstance(matchers, list) assert isinstance(cls_matchers, list) for part, regexp in itertools.chain(matchers, cls_matchers): assert part in _PARTS, repr(part) assert isinstance(regexp, str), repr(regexp) self.remap[part].append(re.compile(regexp)) # Converter is a fn(filename: Text) -> contents: Text. self.converter = kwds.pop('converter', getattr(self, 'converter', None)) super().__init__(**kwds) def identify(self, file): return identify(self.remap, self.converter, file) beangulp-0.2.0/beangulp/mimetypes.py000066400000000000000000000015321474340320100174700ustar00rootroot00000000000000# TODO(blais): Make this only register the new types, and keep the user # referring to the interface directly. There's no real need to 'import *' here. # The root package would automatically register those new types. """The standard Python 'mimetypes' module with some custom registrations for types commonly used for accounting. Import 'from beangulp import mimetypes' instead of 'import mimetypes'. """ from mimetypes import * # noqa: F403 # Register some MIME types used in financial downloads. _extra_mime_types = [ ('text/beancount', '.beancount', '.beans'), # Beancount ledgers. ('application/vnd.intu.qbo', '.qbo'), # Quicken files. ('application/x-ofx', '.qfx', '.ofx'), # Open Financial Exchange files. ] for mime, *extensions in _extra_mime_types: for ext in extensions: add_type(mime, ext, strict=False) # noqa: F405 beangulp-0.2.0/beangulp/petl_utils.py000066400000000000000000000106421474340320100176420ustar00rootroot00000000000000"""Utilities using petl. """ from typing import Optional import datetime import re import petl from beancount.core import data from beancount.core import amount from beancount.core import flags petl.config.look_style = "minimal" petl.config.failonerror = True def table_to_directives( table: petl.Table, currency: str = "USD", filename: Optional[str] = None ) -> data.Entries: """Convert a petl table to Beancount directives. This is intended as a convenience for many simple CSV importers. Your CSV code uses petl to normalize the contents of an imported file to a table, and this routine is called to actually translate that into directives. Required columns of the input 'table' are: date: A datetime.date instance, for the transaction. account: An account string, for the posting. amount: A Decimal instance, the number you want on the posting. Optional columns are: payee: A string, for the transaction's payee. narration: A string, for the transaction's narration. balance: A Decimal, the balance in the account *after* the given transaction. other_account: An account string, for the remainder of the transaction. """ # Ensure the table is sorted in order to produce the final balance. assert table.issorted("date") assert set(table.fieldnames()) >= {"date", "account", "amount"} columns = table.fieldnames() metas = [] for column in columns: match = re.match("meta:(.*)", column) if match: metas.append((column, match.group(1))) # Create transactions. entries = [] filename = filename or f"<{__file__}>" for index, rec in enumerate(table.records()): meta = data.new_metadata(filename, index) units = amount.Amount(rec.amount, currency) tags, links = set(), set() txn = data.Transaction( meta, rec.date, flags.FLAG_OKAY, getattr(rec, "payee", None), getattr(rec, "narration", ""), tags, links, [data.Posting(rec.account, units, None, None, None, None)], ) if hasattr(rec, "other_account") and rec.other_account: txn.postings.append( data.Posting(rec.other_account, None, None, None, None, None) ) link = getattr(rec, "link", None) if link: links.add(link) tag = getattr(rec, "tag", None) if tag: tags.add(tag) for column, key in metas: value = getattr(rec, column, None) if value: meta[key] = value entries.append(txn) if "balance" in columns: # Insert a balance with the final value. meta = data.new_metadata(filename, index + 1) balance_date = rec.date + datetime.timedelta(days=1) entries.append( data.Balance( meta, balance_date, rec.account, amount.Amount(rec.balance, currency), None, None, ) ) return entries def absorb_extra(table: petl.Table, column: str) -> petl.Table: """Absorb extra columns in a specific column. Sadly, some programmers in banks will forego the usage of libraries performing proper escaping of commas. This produces invalid CSV files, where some rows have an extra column or two. This function fixes up those mistakes by specifying a column to absort extra columns. The behavior is as follows: if the header has 7 columns, we assume the rest of the rows in the file should also have 7 columns. If this function is given the column name 'description' and it lives in the 5th column, a row with an abnormal 8 columns will be modified to merge the 5th nd 6th columns. If a row shows up with 9 columns, the 5th column would absort columns 5, 6 and 7. Rows with the normal 7 columns are unaffected. """ header = table.fieldnames() num_expected_cols = len(header) absorbent_col = header.index(column) def absorb(row): row = list(row) if len(row) > num_expected_cols: for _ in range(len(row) - num_expected_cols): row[absorbent_col] = "{}, {}".format( row[absorbent_col], row[absorbent_col + 1] ) del row[absorbent_col + 1] return tuple(row) return table.rowmap(absorb, header=header) beangulp-0.2.0/beangulp/petl_utils_test.py000066400000000000000000000030461474340320100207010ustar00rootroot00000000000000import datetime import decimal import unittest import petl from beancount.parser import cmptest from beangulp import petl_utils class TestPetlUtils(cmptest.TestCase, unittest.TestCase): def test_table_to_directives_minimal(self): table = ( petl.wrap([('date', 'account', 'amount'), ('2021-02-15', 'Assets:Checking', '4.56'), ('2021-02-16', 'Assets:Savings', '107.89')]) .convert('date', lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date()) .convert('amount', decimal.Decimal)) transactions = petl_utils.table_to_directives(table) self.assertEqualEntries(""" 2021-02-15 * Assets:Checking 4.56 USD 2021-02-16 * Assets:Savings 107.89 USD """, transactions) def test_table_to_directives_all(self): table = ( petl.wrap([ ('date', 'payee', 'narration', 'account', 'amount', 'balance'), ('2021-02-15', 'TheStore', 'BuyingSomething', 'Liabilities:Credit', '-4.56', '-73330.00'), ]) .convert('date', lambda x: datetime.datetime.strptime(x, '%Y-%m-%d').date()) .convert(['amount', 'balance'], decimal.Decimal)) transactions = petl_utils.table_to_directives(table) self.assertEqualEntries(""" 2021-02-15 * "TheStore" "BuyingSomething" Liabilities:Credit -4.56 USD 2021-02-16 balance Liabilities:Credit -73330.00 USD """, transactions) beangulp-0.2.0/beangulp/scripts_utils.py000066400000000000000000000005131474340320100203610ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016,2018 Martin Blais" __license__ = "GNU GPLv2" from beangulp import Ingest def ingest(importers, hooks=None): # This function is provided for backward compatibility with the # ``beancount.ingest`` framework. It will be removed eventually. main = Ingest(importers, hooks) main() beangulp-0.2.0/beangulp/similar.py000066400000000000000000000170241474340320100171170ustar00rootroot00000000000000"""Identify similar entries. This can be used during import in order to identify and flag duplicate entries. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" from decimal import Decimal from typing import Callable, Optional import collections import datetime import re from beancount.core.number import ZERO from beancount.core.number import ONE from beancount.core import data from beancount.core import amount from beancount.core import interpolate try: from functools import cache except ImportError: from functools import lru_cache cache = lru_cache(maxsize=None) def find_similar_entries(entries, existing_entries, cmp=None, window_days=2): """Find which entries from a list are potential duplicates of a set. The existing_entries array must be sorted by date. If there are multiple entries in existing_entries matching an entry in entries, only the first match is returned. Args: entries: The list of entries to classify as duplicate or note. existing_entries: The list of entries against which to match. cmp: A functor used to establish the similarity of two entries. window_days: The number of days (inclusive) before or after to scan the entries to classify against. Returns: A list of (entry, existing_entry) tuples where entry is from entries and is deemed to be a duplicate of existing_entry, from existing_entries. """ window_head = datetime.timedelta(days=window_days) window_tail = datetime.timedelta(days=window_days + 1) if cmp is None: cmp = heuristic_comparator() # For each of the new entries, look at existing entries at a nearby date. duplicates = [] if existing_entries is not None: for entry in data.filter_txns(entries): for existing_entry in data.filter_txns( data.iter_entry_dates( existing_entries, entry.date - window_head, entry.date + window_tail ) ): if cmp(entry, existing_entry): duplicates.append((entry, existing_entry)) break return duplicates class hashable: def __init__(self, obj): self.obj = obj def __hash__(self): return id(self.obj) def __getattr__(self, name): return getattr(self.obj, name) Comparator = Callable[[data.Directive, data.Directive], bool] def heuristic_comparator( max_date_delta: Optional[datetime.timedelta] = None, epsilon: Optional[Decimal] = None ) -> Comparator: """Generic comparison function generator. Two transactions are deemed similar if - their dates are within a close range of each other (e.g. 2 days), if specified with `max_date_delta`, - amounts on postings corresponding to the same account are within some fraction of each other (default: 5%), and - the set of accounts of the two transactions are the same or one is a subset of the other. Args: max_date_delta: A timedelta datetime difference within which two transactions may be considered similar. epsilon: A Decimal fraction representing how close the amounts are required to be of each other. For example, Decimal("0.01") for 1%. Returns: A comparator predicate accepting two directives and returning a bool. """ if epsilon is None: epsilon = Decimal("0.05") def cmp(entry1: data.Directive, entry2: data.Directive) -> bool: """Compare two entries. Implement a heuristic method to determine if two transactions are similar enough to be considered duplicates. """ # This comparator needs to be able to handle Transaction # instances which are incomplete on one side, which have # slightly different dates, or potentially postings with # slightly different amounts. if not isinstance(entry1, data.Transaction) or not isinstance( entry2, data.Transaction ): return False # Check the date difference. if max_date_delta is not None: delta = ( (entry1.date - entry2.date) if entry1.date > entry2.date else (entry2.date - entry1.date) ) if delta > max_date_delta: return False amounts1 = amounts_map_cached(hashable(entry1)) amounts2 = amounts_map_cached(hashable(entry2)) # Look for amounts on common accounts. common_keys = set(amounts1) & set(amounts2) for key in sorted(common_keys): # Compare the amounts. number1 = amounts1[key] number2 = amounts2[key] if number1 == ZERO and number2 == ZERO: break diff = abs((number1 / number2) if number2 != ZERO else (number2 / number1)) if diff == ZERO: return False if diff < ONE: diff = ONE / diff if (diff - ONE) < epsilon: break else: return False # Here, we have found at least one common account with a close # amount. Now, we require that the set of accounts are equal or that # one be a subset of the other. accounts1 = {posting.account for posting in entry1.postings} accounts2 = {posting.account for posting in entry2.postings} return accounts1.issubset(accounts2) or accounts2.issubset(accounts1) return cmp # Old alias to the heuristic comparator kept for backwards compatibility. comparator = heuristic_comparator def same_link_comparator(regex: Optional[str] = None) -> Comparator: """Comparison function generator that checks if two directives share a link. You can use this if you have a source of transactions that consistently defined and unique transaction which it produces as a link. The matching of these transactions will be more precise than the heuristic no dates and amounts used by default, if you keep the links in your ledger. You can further restrict the set of links that are compared if you provide a regex. This can be useful if your importer produces multiple other links. Args: regex: An optional regular expression used to filter the links. Returns: A comparator predicate accepting two directives and returning a bool. """ def cmp(entry1: data.Directive, entry2: data.Directive) -> bool: """Compare two entries by common link.""" if not isinstance(entry1, data.Transaction) or not isinstance( entry2, data.Transaction ): return False links1 = entry1.links links2 = entry2.links if regex: links1 = {link for link in links1 if re.match(regex, link)} links2 = {link for link in links2 if re.match(regex, link)} return bool(links1 & links2) return cmp def amounts_map(entry): """Compute a mapping of (account, currency) -> Decimal balances. Args: entry: A Transaction instance. Returns: A dict of account -> Amount balance. """ amounts = collections.defaultdict(Decimal) for posting in entry.postings: # Skip interpolated postings. if posting.meta and interpolate.AUTOMATIC_META in posting.meta: continue currency = isinstance(posting.units, amount.Amount) and posting.units.currency if isinstance(currency, str): key = (posting.account, currency) amounts[key] += posting.units.number return amounts amounts_map_cached = cache(amounts_map) beangulp-0.2.0/beangulp/similar_test.py000066400000000000000000000154211474340320100201550ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" from decimal import Decimal as D import datetime import re from beancount.core import data from beancount.parser import cmptest from beancount.parser import parser from beancount import loader from beangulp import similar class TestDups(cmptest.TestCase): @loader.load_doc() def test_find_similar_entries(self, entries, _, __): """ plugin "beancount.plugins.auto_accounts" 2016-01-03 * Expenses:Tips 1.03 USD Assets:Other 2016-01-04 * Expenses:Coffee 1.04 USD Assets:Other 2016-01-05 * Expenses:Restaurant 1.05 USD Assets:Other 2016-01-06 * Expenses:Groceries 1.06 USD Assets:Other 2016-01-07 * Expenses:Alcohol 1.07 USD Assets:Other 2016-01-08 * Expenses:Smoking 1.08 USD Assets:Other 2016-01-09 * Expenses:Taxi 1.09 USD Assets:Other """ new_entries, _, __ = loader.load_string( """ plugin "beancount.plugins.auto_accounts" 2016-01-06 * Expenses:Groceries 1.06 USD Assets:Other """ ) for days, num_comparisons in [(0, 1), (1, 1), (2, 1)]: duplicates = similar.find_similar_entries( new_entries, entries, lambda e1, e2: True, window_days=days ) self.assertEqual(num_comparisons, len(duplicates)) duplicates = similar.find_similar_entries( new_entries, entries, lambda e1, e2: False, window_days=days ) self.assertEqual(0, len(duplicates)) @loader.load_doc() def test_find_similar_entries__multiple_matches(self, entries, _, __): """ plugin "beancount.plugins.auto_accounts" 2016-02-01 * "A" Assets:Account1 10.00 USD Assets:Account2 -10.00 USD 2016-02-02 * "B" Assets:Account1 10.00 USD Assets:Account2 -10.00 USD 2016-02-03 * "C" Assets:Account1 10.00 USD Assets:Account2 -10.00 USD 2016-02-04 * "D" Assets:Account1 10.00 USD Assets:Account2 -10.00 USD 2016-02-05 * "D" Assets:Account1 10.00 USD Assets:Account2 -10.00 USD """ # Test it with a single entry. new_entries = list(data.filter_txns(entries))[2:3] duplicates = similar.find_similar_entries(new_entries, entries, window_days=1) self.assertEqual(1, len(duplicates)) self.assertEqual(new_entries[0], duplicates[0][0]) # Test it with multiple duplicate entries. new_entries = list(data.filter_txns(entries))[1:4] duplicates = similar.find_similar_entries(new_entries, entries, window_days=1) self.assertEqual(len(new_entries), len(duplicates)) @parser.parse_doc(allow_incomplete=True) def test_amounts_map(self, entries, _, __): """ plugin "beancount.plugins.auto_accounts" 2016-01-03 * Expenses:Alcohol 20.00 USD Expenses:Tips 1.03 USD Assets:Other 2016-01-03 * Expenses:Tips 1.01 USD Expenses:Tips 1.02 USD Assets:Other """ txns = list(data.filter_txns(entries)) amap = similar.amounts_map(txns[0]) self.assertEqual( { ("Expenses:Tips", "USD"): D("1.03"), ("Expenses:Alcohol", "USD"): D("20.00"), }, amap, ) amap = similar.amounts_map(txns[1]) self.assertEqual({("Expenses:Tips", "USD"): D("2.03")}, amap) class TestHeuristicComparator(cmptest.TestCase): def setUp(self): self.comparator = similar.heuristic_comparator(datetime.timedelta(days=2)) @loader.load_doc() def test_simple(self, entries, _, __): """ plugin "beancount.plugins.auto_accounts" 2016-01-03 * "Base reservation" ^base Expenses:Alcohol 20.00 USD Expenses:Tips 1.03 USD Assets:Other 2016-01-03 * "Similar amount within bounds" ^in-bounds Expenses:Alcohol 20.99 USD Assets:Other 2016-01-03 * "Similar amount out of bounds" ^out-bounds Expenses:Alcohol 21.00 USD Assets:Other 2016-01-06 * "Date too far" ^too-late Expenses:Alcohol 20.00 USD Expenses:Tips 1.03 USD Assets:Other 2016-01-03 * "Non-overlapping accounts" ^non-accounts Expenses:Alcohol 20.00 USD Expenses:Tips 1.03 USD Assets:SomethingElse """ txns = list(data.filter_txns(entries)) def compare(expected, link1, link2): self.assertEqual( expected, self.comparator( next(txn for txn in txns if link1 in txn.links), next(txn for txn in txns if link2 in txn.links), ), ) compare(True, "base", "base") compare(True, "base", "in-bounds") compare(False, "base", "out-bounds") compare(False, "base", "too-late") compare(False, "base", "non-accounts") class TestSamelinkComparator(cmptest.TestCase): def setUp(self): self.comparator = similar.same_link_comparator(regex=r"\d+$") @loader.load_doc() def test_simple(self, entries, _, __): """ 2016-01-01 open Expenses:Alcohol 2016-01-01 open Expenses:Food 2016-01-01 open Expenses:Tips 2016-01-01 open Assets:Other 2016-01-03 * "Base" ^43567284654849564 Expenses:Alcohol 20.00 USD Expenses:Tips 1.03 USD Assets:Other 2016-01-04 * "Matching" ^43567284654849564 Expenses:Food 25.00 USD Assets:Other 2016-02-01 * "Matching with far date" ^43567284654849564 Expenses:Food 25.00 USD Assets:Other 2016-01-04 * "No links" Expenses:Food 25.00 USD Assets:Other 2016-01-04 * "Not the same link" ^too-late Expenses:Food 25.00 USD Assets:Other 2016-01-04 * "Difference number, matches regex" ^93580348349894834 Expenses:Food 25.00 USD Assets:Other """ txns = list(data.filter_txns(entries)) for txn in filter(lambda t: re.match("Matching", t.narration), txns): self.assertEqual(True, self.comparator(txns[0], txn)) for txn in filter(lambda t: not re.match("Matching", t.narration), txns[1:]): self.assertEqual(False, self.comparator(txns[0], txn)) beangulp-0.2.0/beangulp/testing.py000066400000000000000000000231301474340320100171270ustar00rootroot00000000000000"""Implementation of testing and generate functionality.""" from os import path from typing import Callable, List, Optional, TextIO, Tuple, Union import datetime import difflib import io import os import sys import warnings import click from beancount.parser import printer from beancount.core import data import beangulp from beangulp.importer import Importer, ImporterProtocol from beangulp import extract from beangulp import utils def write_expected(outfile: TextIO, account: data.Account, date: Optional[datetime.date], name: Optional[str], entries: data.Entries): """Produce the expected output file. Args: outfile: The file object where to write. account: The account name produced by the importer. date: The date of the downloads file, produced by the importer. name: The filename for filing, produced by the importer. entries: The list of entries extracted by the importer. """ date = date.isoformat() if date else '' name = name or '' print(f';; Account: {account}', file=outfile) print(f';; Date: {date}', file=outfile) print(f';; Name: {name}', file=outfile) printer.print_entries(entries, file=outfile) def write_expected_file(filepath: str, *data, force: bool = False): """Writes out the expected file.""" mode = 'w' if force else 'x' with open(filepath, mode) as expfile: write_expected(expfile, *data) def compare_expected(filepath: str, *data) -> List[str]: """Compare the expected file with extracted data.""" with io.StringIO() as buffer: write_expected(buffer, *data) # rewind to the beginning of the stream buffer.seek(0) lines_imported = buffer.readlines() with open(filepath, 'r') as infile: lines_expected = infile.readlines() diff = difflib.unified_diff(lines_expected, lines_imported, tofile='expected.beancount', fromfile='imported.beancount') return list(diff) def run_importer(importer: Importer, document: str) -> Tuple[data.Account, Optional[datetime.date], Optional[str], data.Entries]: """Run the various importer methods on the given cached file.""" account = importer.account(document) date = importer.date(document) name = importer.filename(document) entries = extract.extract_from_file(importer, document, []) return account, date, name, entries @click.command('test') @click.argument('documents', nargs=-1, type=click.Path(exists=True, resolve_path=True)) @click.option('--expected', '-e', metavar='DIR', type=click.Path(file_okay=False, resolve_path=True), help="Directory containing the expecrted output files.") @click.option('--verbose', '-v', count=True, help="Enable verbose output.") @click.option('--quiet', '-q', count=True, help="Suppress all output.") @click.option('--failfast', '-x', is_flag=True, help="Stop at the first test failure.") @click.pass_obj def _test(ctx, documents: List[str], expected: str, verbose: int, quiet: int, failfast: bool): """Test the importer. Run the importer on all DOCUMENTS and verify that it produces the desired output. The desired output is stored in Beancount ledger files with a header containing additional metadata and can be generated with the "generate" command. The name of the desired output files is derived appending a ".beancount" suffix to the name of the input files, and are searched in the same directory where the input document is located, unless a different location is specified through the "--expected DIR" option. DOCUMENTS can be files or directories. Directories are walked and the importer is called on all files with names not ending in ".beancount". All and only the documents for which a desired output file exists must be positively identify by the importer. """ return _run(ctx, documents, expected, verbose, quiet, failfast=failfast) @click.command('generate') @click.argument('documents', nargs=-1, type=click.Path(exists=True, resolve_path=True)) @click.option('--expected', '-e', metavar='DIR', type=click.Path(file_okay=False, resolve_path=True), help="Directory containing the expecrted output files.") @click.option('--verbose', '-v', count=True, help="Enable verbose output.") @click.option('--quiet', '-q', count=True, help="Suppress all output.") @click.option('--force', '-f', is_flag=True, help='Alow to overwrite existing files.') @click.pass_obj def _generate(ctx, documents: List[str], expected: str, verbose: int, quiet: int, force: bool): """Generate expected files for tests. Run the importer on all DOCUMENTS and save the import results in Beancount ledger files with an header containing additional metadata that can be used to as regression tests for the importer. The name of the desired output files is derived appending a ".beancount" suffix to the name of the input files, and are written in the same directory where the input document is located, unless a different location is specified through the "--expected DIR" option. DOCUMENTS can be files or directories. Directories are walked and the importer is called on all files with names not ending in ".beancount". """ return _run(ctx, documents, expected, verbose, quiet, generate=True, force=force) def _run(ctx, documents: List[str], expected: str, verbose: int, quiet: int, generate: bool = False, failfast: bool = False, force: bool = False): """Implement the test and generate commands.""" assert len(ctx.importers) == 1 importer = ctx.importers[0] verbosity = verbose - quiet log = utils.logger(verbosity) failures = 0 for doc in utils.walk(documents): if doc.endswith('.beancount'): continue # Unless verbose mode is enabled, do not output a newline so # the test result is printed on the same line as the test # document filename. log(f'* {doc}', nl=verbosity > 0) # Compute the path to the expected output file. expected_filename = f"{doc}.beancount" if expected: expected_filename = path.join(expected, path.basename(expected_filename)) # Run the importer's identify() method. if importer.identify(doc): account, date, name, entries = run_importer(importer, doc) log(f' {expected_filename}', 1) if account is None: failures += 1 log(' ERROR', fg='red') log(' ValueError: account() should not return None') continue log(' {}/{:%Y-%m-%d}-{}'.format( account.replace(":", "/"), date or utils.getmdate(doc), name or path.basename(doc)), 1) if generate: try: write_expected_file(expected_filename, account, date, name, entries, force=force) except FileExistsError as ex: failures += 1 log(' ERROR', fg='red') log(' FileExistsError: {}'.format(ex.filename)) continue log(' OK', fg='green') continue try: diff = compare_expected(expected_filename, account, date, name, entries) except FileNotFoundError: # The importer has positively identified a document # for which there is no expecred output file. failures += 1 log(' ERROR', fg='red') log(' ExpectedOutputFileNotFound') continue if diff: # Test failure. Log an error. failures += 1 log(' ERROR', fg='red') if verbosity >= 0: sys.stdout.writelines(diff) sys.stdout.write(os.linesep) continue log(' PASSED', fg='green') elif path.exists(expected_filename): # The importer has not identified a document it should have. failures += 1 log(' ERROR', fg='red') log(' DocumentNotIdentified') else: # Ignore files that are not positively identified by the # importer and for which there is no expected output file. log(' IGNORED') if failfast and failures: break if failures: sys.exit(1) def wrap(importer: Union[Importer, ImporterProtocol]) -> Callable[[], None]: """Wrap a single importer for ingestion.""" cli = beangulp.Ingest([importer]).cli cli.help = importer.__doc__ cli.add_command(_test) cli.add_command(_generate) return cli def main(importer: Union[Importer, ImporterProtocol]): """Call main program on a single importer. This is the main entry point.""" if not sys.warnoptions: # Even if DeprecationWarnings are ignored by default print # them anyway unless other warnings settings are specified by # the -W Python command line flag. warnings.simplefilter('default') main = wrap(importer) main() beangulp-0.2.0/beangulp/tests/000077500000000000000000000000001474340320100162435ustar00rootroot00000000000000beangulp-0.2.0/beangulp/tests/__init__.py000066400000000000000000000000421474340320100203500ustar00rootroot00000000000000from . import utils # noqa: F401 beangulp-0.2.0/beangulp/tests/archive.rst000066400000000000000000000070121474340320100204160ustar00rootroot00000000000000Setup ----- >>> from os import mkdir, path, unlink >>> from shutil import rmtree >>> from tempfile import mkdtemp >>> import click.testing >>> import beangulp >>> from beangulp.tests.utils import Importer An importer to create error conditions: >>> class ErrorImporter(Importer): ... ... def identify(self, filepath): ... name = path.basename(filepath) ... if name == 'error.foo': ... return True ... return False ... ... def filename(self, filepath): ... return 'bbb.csv' Test harness: >>> importers = [ ... Importer('test.ImporterA', 'Assets:Tests', 'text/csv'), ... ErrorImporter('test.ImporterB', 'Assets:Tests', None), ... ] >>> runner = click.testing.CliRunner() >>> def run(*args): ... ingest = beangulp.Ingest(importers) ... return runner.invoke(ingest.cli, args, catch_exceptions=False) Create a dowloads and a documents directory: >>> temp = mkdtemp() >>> downloads = path.join(temp, 'downloads') >>> documents = path.join(temp, 'documents') >>> mkdir(downloads) >>> mkdir(documents) Tests ----- The basics: >>> r = run('archive', '--help') >>> r.exit_code 0 >>> print(r.output) Usage: beangulp archive [OPTIONS] [SRC]... Test with an empty downloads directory: >>> r = run('archive', downloads) >>> r.exit_code 0 >>> print(r.output) Add some documents: >>> fnames = ['aaa.txt', 'bbb.csv', 'zzz.txt'] >>> for fname in fnames: ... with open(path.join(downloads, fname), 'w') as f: ... pass Run in dry-run mode: >>> r = run('archive', downloads, '-o', documents, '-n') >>> r.exit_code 0 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK .../documents/Assets/Tests/1970-01-01.bbb.csv * .../downloads/zzz.txt No files have actually been moved: >>> path.exists(path.join(downloads, 'bbb.csv')) True >>> path.exists(path.join(documents, 'Assets/Tests/1970-01-01.bbb.csv')) False Now for real: >>> r = run('archive', downloads, '-o', documents) >>> r.exit_code 0 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK .../documents/Assets/Tests/1970-01-01.bbb.csv * .../downloads/zzz.txt >>> path.exists(path.join(downloads, 'bbb.csv')) False >>> path.exists(path.join(documents, 'Assets/Tests/1970-01-01.bbb.csv')) True Trying to move a documents over an exisiting file: >>> with open(path.join(downloads, 'bbb.csv'), 'w') as f: ... pass >>> r = run('archive', downloads, '-o', documents) >>> r.exit_code 1 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... ERROR .../documents/Assets/Tests/1970-01-01.bbb.csv Destination file already exists. * .../downloads/zzz.txt # Errors detected: documents will not be filed. Cleanup documents directory: >>> rmtree(documents) >>> mkdir(documents) Collision in destination filename: >>> fnames = ['aaa.txt', 'bbb.csv', 'zzz.txt', 'error.foo'] >>> for fname in fnames: ... with open(path.join(downloads, fname), 'w') as f: ... pass >>> r = run('archive', downloads, '-o', documents) >>> r.exit_code 1 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK .../documents/Assets/Tests/1970-01-01.bbb.csv * .../downloads/error.foo ... ERROR .../documents/Assets/Tests/1970-01-01.bbb.csv Collision in destination file path. * .../downloads/zzz.txt # Errors detected: documents will not be filed. Cleanup ------- >>> rmtree(temp) beangulp-0.2.0/beangulp/tests/doctests_test.py000066400000000000000000000013451474340320100215070ustar00rootroot00000000000000"""Test loader to make unittest test discovery pick up the doctests.""" import doctest import unittest def load_tests(loader, tests, pattern): suite = unittest.TestSuite() suite.addTest( doctest.DocFileSuite( 'archive.rst', 'extract.rst', 'identify.rst', 'testing.rst', optionflags=( doctest.ELLIPSIS | doctest.NORMALIZE_WHITESPACE | doctest.REPORT_NDIFF | # Display only the first failed test. Note that all # tests are run, thus the cleanup at the end of the # doctests file is still executed. doctest.REPORT_ONLY_FIRST_FAILURE))) return suite beangulp-0.2.0/beangulp/tests/extract.rst000066400000000000000000000103411474340320100204460ustar00rootroot00000000000000Setup ----- >>> import pathlib >>> from os import mkdir, path, unlink >>> from shutil import rmtree >>> from tempfile import mkdtemp >>> import click.testing >>> import beangulp >>> from beangulp.tests.utils import Importer, IdentityImporter An importer to create error conditions: >>> class ErrorImporter(Importer): ... ... def extract(self, filepath, existing): ... name = path.basename(filepath) ... raise ValueError(name) Test harness: >>> importers = [ ... Importer('test.ImporterA', 'Assets:Tests', 'text/csv'), ... ErrorImporter('test.ImporterE', 'Assets:Tests', None), ... IdentityImporter('test.Identity1', 'Assets:Tests', '*one.beans'), ... IdentityImporter('test.Identity2', 'Assets:Tests', '*two.beans'), ... ] >>> runner = click.testing.CliRunner() >>> def run(*args): ... ingest = beangulp.Ingest(importers) ... return runner.invoke(ingest.cli, args, catch_exceptions=False) Create a dowloads directory: >>> temp = mkdtemp() >>> downloads = path.join(temp, 'downloads') >>> mkdir(downloads) Tests ----- The basics: >>> r = run('extract', '--help') >>> r.exit_code 0 >>> print(r.output) Usage: beangulp extract [OPTIONS] [SRC]... Test with an empty downloads directory: >>> r = run('extract', downloads) >>> r.exit_code 0 >>> print(r.output) Add some documents: >>> fnames = ['aaa.txt', 'bbb.csv', 'zzz.txt'] >>> for fname in fnames: ... with open(path.join(downloads, fname), 'w') as f: ... pass >>> output = path.join(temp, 'output.beancount') >>> r = run('extract', downloads, '-o', output) >>> r.exit_code 0 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK * .../downloads/zzz.txt Check the output file: >>> with open(output, 'r') as f: ... extracted = f.read() >>> print(extracted) ;; -*- mode: beancount -*- **** .../downloads/bbb.csv Test importer raising an error: >>> with open(path.join(downloads, 'error.foo'), 'w') as f: ... pass >>> output = path.join(temp, 'output.beancount') >>> r = run('extract', downloads, '-o', output) >>> r.exit_code 1 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK * .../downloads/error.foo ... ERROR Exception in importer code. Traceback (most recent call last): ... ValueError: error.foo * .../downloads/zzz.txt Check the output file: >>> with open(output, 'r') as f: ... extracted = f.read() >>> print(extracted) ;; -*- mode: beancount -*- **** .../downloads/bbb.csv Cleanup: >>> rmtree(downloads) >>> mkdir(downloads) Deduplication ------------- Test the identity importer: >>> existing = path.join(temp, 'existing.beancount') >>> _ = pathlib.Path(downloads).joinpath('one.beans').write_text(""" ... 2023-01-01 * "Test" ... Assets:Tests 2 TESTS ... """) >>> r = run('extract', downloads, '-o', existing) >>> r.exit_code 0 >>> print(r.output) * .../downloads/one.beans ... OK >>> print(pathlib.Path(existing).read_text()) ;; -*- mode: beancount -*- **** .../downloads/one.beans 2023-01-01 * "Test" Assets:Tests 2 TESTS Importing again the same file results in entries marked as duplicates: >>> r = run('extract', downloads, '-o', output, '-e', existing) >>> r.exit_code 0 >>> print(r.output) * .../downloads/one.beans ... OK >>> print(pathlib.Path(output).read_text()) ;; -*- mode: beancount -*- **** .../downloads/one.beans ; duplicate of .../existing.beancount:5 ; 2023-01-01 * "Test" ; Assets:Tests 2 TESTS >>> _ = pathlib.Path(downloads).joinpath('two.beans').write_text(""" ... 2023-01-01 * "Test" ... Assets:Tests 2 TESTS ... """) >>> r = run('extract', downloads, '-o', output) >>> r.exit_code 0 >>> print(pathlib.Path(output).read_text()) ;; -*- mode: beancount -*- **** .../downloads/one.beans 2023-01-01 * "Test" Assets:Tests 2 TESTS **** .../downloads/two.beans ; .../downloads/one.beans:2 ; 2023-01-01 * "Test" ; Assets:Tests 2 TESTS Cleanup ------- >>> rmtree(temp) beangulp-0.2.0/beangulp/tests/identify.rst000066400000000000000000000053301474340320100206110ustar00rootroot00000000000000Setup ----- >>> from os import mkdir, path, unlink >>> from shutil import rmtree >>> from tempfile import mkdtemp >>> import click.testing >>> import beangulp >>> from beangulp.tests.utils import Importer An importer to create error conditions: >>> class ErrorImporter(Importer): ... ... def identify(self, filepath): ... name = path.basename(filepath) ... # An exception raised from importer code. ... if name == 'error.txt': ... raise ValueError(name) ... # A collision in identification. ... if name == 'error.csv': ... return True ... return False Test harness: >>> importers = [ ... Importer('test.ImporterA', 'Assets:Tests', 'text/csv'), ... ErrorImporter('test.ImporterB', 'Assets:Tests', None), ... ] >>> runner = click.testing.CliRunner() >>> def run(*args): ... ingest = beangulp.Ingest(importers) ... return runner.invoke(ingest.cli, args, catch_exceptions=False) Create a dowloads directory: >>> temp = mkdtemp() >>> downloads = path.join(temp, 'downloads') >>> mkdir(downloads) Tests ----- The basics: >>> r = run('identify', '--help') >>> r.exit_code 0 >>> print(r.output) Usage: beangulp identify [OPTIONS] [SRC]... Test with an empty downloads directory: >>> r = run('identify', downloads) >>> r.exit_code 0 >>> print(r.output) Add some documents: >>> fnames = ['aaa.txt', 'bbb.csv', 'zzz.txt'] >>> for fname in fnames: ... with open(path.join(downloads, fname), 'w') as f: ... pass >>> r = run('identify', downloads) >>> r.exit_code 0 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK test.ImporterA * .../downloads/zzz.txt Error conditions ---------------- Exception raised in importer code: >>> with open(path.join(downloads, 'error.txt'), 'w') as f: ... pass >>> r = run('identify', downloads) >>> r.exit_code 1 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK test.ImporterA * .../downloads/error.txt ERROR Exception in importer code. Traceback (most recent call last): ... ValueError: error.txt * .../downloads/zzz.txt >>> unlink(path.join(downloads, 'error.txt')) Two importers matching the same document: >>> with open(path.join(downloads, 'error.csv'), 'w') as f: ... pass >>> r = run('identify', downloads) >>> r.exit_code 1 >>> print(r.output) * .../downloads/aaa.txt * .../downloads/bbb.csv ... OK test.ImporterA * .../downloads/error.csv ERROR test.ImporterA test.ImporterB Document identified by more than one importer. * .../downloads/zzz.txt Cleanup ------- >>> rmtree(temp) beangulp-0.2.0/beangulp/tests/testing.rst000066400000000000000000000074001474340320100204530ustar00rootroot00000000000000Setup ----- >>> from datetime import date >>> from os import mkdir, path, rename, unlink >>> from shutil import rmtree >>> from tempfile import mkdtemp >>> from beangulp.tests.utils import Importer >>> import click.testing Import the module under test: >>> import beangulp.testing Test harness: >>> importer = Importer('test.Importer', 'Assets:Tests', 'text/csv') >>> runner = click.testing.CliRunner() >>> def run(*args): ... func = beangulp.testing.wrap(importer) ... return runner.invoke(func, args, catch_exceptions=False) Tests ----- Check the basics: >>> r = run() >>> r.exit_code 0 >>> print(r.output) Usage: beangulp [OPTIONS] COMMAND [ARGS]... >>> r = run('test', '--help') >>> r.exit_code 0 >>> print(r.output) Usage: beangulp test [OPTIONS] [DOCUMENTS]... >>> r = run('test') >>> r.exit_code 0 >>> print(r.output) Create a documents directory: >>> temp = mkdtemp() >>> documents = path.join(temp, 'documents') >>> mkdir(documents) Poulate it with a file that should be ignored: >>> with open(path.join(documents, 'test.txt'), 'w') as f: ... pass The test harness should report this file as ignored and report success: >>> r = run('test', documents) >>> r.exit_code 0 >>> print(r.output) * .../documents/test.txt IGNORED and no expected output file should be generated for it: >>> r = run('generate', documents) >>> r.exit_code 0 >>> print(r.output) * .../documents/test.txt IGNORED >>> unlink(path.join(documents, 'test.txt')) Try the same with a file that should be recognized by the importer. When there is no epxected output file the test harness should report a test error: >>> with open(path.join(documents, 'test.csv'), 'w') as f: ... pass >>> r = run('test', documents) >>> r.exit_code 1 >>> print(r.output) * .../documents/test.csv ERROR ExpectedOutputFileNotFound Generate the expected output file: >>> r = run('generate', documents) >>> r.exit_code 0 >>> print(r.output) * .../documents/test.csv OK Now the test should succeed: >>> r = run('test', documents) >>> r.exit_code 0 >>> print(r.output) * .../documents/test.csv PASSED Overwriting the expected output file is an error: >>> r = run('generate', documents) >>> r.exit_code 1 >>> print(r.output) * .../documents/test.csv ERROR FileExistsError: .../test.csv.beancount unless the --force options is specified: >>> r = run('generate', documents, '--force') >>> r.exit_code 0 >>> print(r.output) * .../documents/test.csv OK Put back a file that should be ignored and verify that it is: >>> with open(path.join(documents, 'test.txt'), 'w') as f: ... pass >>> r = run('test', documents) >>> r.exit_code 0 >>> print(r.output) * .../documents/test.csv PASSED * .../documents/test.txt IGNORED >>> unlink(path.join(documents, 'test.txt')) Altering the expected output file should result in a test error: >>> filename = path.join(documents, 'test.csv.beancount') >>> with open(filename, 'a') as f: ... f.write('FAIL') 4 >>> r = run('test', documents) >>> r.exit_code 1 >>> print(r.output) * .../documents/test.csv ERROR --- imported.beancount +++ expected.beancount @@ -1,4 +1,3 @@ ;; Account: Assets:Tests ;; Date: 1970-01-01 ;; Name: -FAIL When the importer does not positively identify a document that should, a test error is reported: >>> rename(path.join(documents, 'test.csv'), path.join(documents, 'test.foo')) >>> rename(path.join(documents, 'test.csv.beancount'), path.join(documents, 'test.foo.beancount')) >>> r = run('test', documents) >>> r.exit_code 1 >>> print(r.output) * .../documents/test.foo ERROR DocumentNotIdentified Cleanup ------- >>> rmtree(temp) beangulp-0.2.0/beangulp/tests/utils.py000066400000000000000000000027761474340320100177710ustar00rootroot00000000000000import fnmatch from datetime import date from beancount.parser import parser from beangulp import importer from beangulp import mimetypes class Importer(importer.Importer): def __init__(self, name, account, mimetype): self._name = name self._account = account self._mimetype = mimetype @property def name(self): return self._name def identify(self, filepath): mimetype, encoding = mimetypes.guess_type(filepath, False) return mimetype == self._mimetype def date(self, filepath): return date(1970, 1, 1) def account(self, filepath): return self._account def filename(self, filepath): return None def extract(self, filepath, existing): return [] def deduplicate(self, entries, existing): # Opt-out from the default deduplication mechanism. return entries class IdentityImporter(importer.Importer): def __init__(self, name, account, match): self._name = name self._account = account self._match = match @property def name(self): return self._name def identify(self, filepath): return fnmatch.fnmatch(filepath, self._match) def date(self, filepath): return date(1970, 1, 1) def account(self, filepath): return self._account def filename(self, filepath): return None def extract(self, filepath, existing): entries, errors, options = parser.parse_file(filepath) return entries beangulp-0.2.0/beangulp/utils.py000066400000000000000000000124701474340320100166170ustar00rootroot00000000000000from decimal import Decimal from os import path from typing import Iterator, Sequence, Union, Set, Optional, Dict import datetime import collections import decimal import hashlib import logging import os import re import click from beangulp import mimetypes class DefaultDictWithKey(collections.defaultdict): """A version of defaultdict whose factory accepts the key as an argument. Note: collections.defaultdict would be improved by supporting this directly, this is a common occurrence. """ def __missing__(self, key): self[key] = value = self.default_factory(key) return value def getmdate(filepath: str) -> datetime.date: """Return file modification date.""" mtime = path.getmtime(filepath) return datetime.datetime.fromtimestamp(mtime).date() def logger(verbosity: int = 0, err: bool = False): """Convenient logging method factory.""" color = False if os.getenv('TERM', '') in ('', 'dumb') else None def log(msg, level=0, err=err, **kwargs): if level <= verbosity: click.secho(msg, color=color, err=err, **kwargs) return log def walk(paths: Sequence[str]) -> Iterator[str]: """Yield all the files under paths. Takes a sequence of file or directory paths. Directories are traversed with os.walk() and complete file paths are returned joining filenames to the root directory path. Other elements of the list are assumed to be file paths and returned unchanged. Args: paths: List of filesystems paths. Yields: Filesystem paths of all the files under paths. """ for file_or_dir in paths: if path.isdir(file_or_dir): for root, dirs, files in os.walk(file_or_dir): for filename in sorted(files): yield path.join(root, filename) continue yield file_or_dir def sha1sum(filepath: str) -> str: """Compute hash of the file at filepath.""" with open(filepath, 'rb') as fd: return hashlib.sha1(fd.read()).hexdigest() def is_mimetype(filepath: str, check_mimetypes: Union[str, Set[str]], regexp: Optional[bool] = False) -> bool: """Check if a file is of one of many mimetypes.""" if isinstance(check_mimetypes, str): check_mimetypes = {check_mimetypes,} mtype, _ = mimetypes.guess_type(filepath) if mtype is None: return False return (any(re.fullmatch(r, mtype) for r in check_mimetypes) if not regexp else (mtype in check_mimetypes)) def search_file_regexp(filepath: str, *regexps: str, nbytes: Optional[int] = None, encoding: Optional[str] = None) -> bool: """Check if the header of the file matches the given regexp.""" with open(filepath, encoding=encoding) as infile: # Note: Don't convert just to match on the contents. try: contents = infile.read(nbytes) except UnicodeDecodeError as exc: # The encoding wasn't right, don't match. logging.warning(f"Error searching for regexp in '{filepath}': {exc}") return False else: return any(re.search(regexp, contents) for regexp in regexps) def parse_amount(string: str)-> decimal.Decimal: """Convert an amount with parens and dollar sign to Decimal.""" if string is None: return Decimal(0) string = string.strip() if not string: return Decimal(0) match = re.match(r"\((.*)\)", string) if match: string = match.group(1) sign = -1 else: sign = 1 cstring = string.replace("-$", "$-").strip(' $').replace(',', '') try: return Decimal(cstring) * sign except decimal.InvalidOperation as exc: raise decimal.InvalidOperation(f"Invalid conversion of {cstring!r}") from exc def validate_accounts(required_accounts: Dict[str, str], provided_accounts: Dict[str, str]): """Check a dict of provided account names against a specification of required ones. Args: required_accounts: A dict of declarations of required values. provided_accounts: A config dict of actual values on an importer. Raises: ValueError: If the configuration is invalid. """ # Note: As an extension, we could provide the existing ledger and try to # validate the non-template names against the list of accounts declared in # it. provided_keys = set(provided_accounts) required_keys = set(required_accounts) for key in (required_keys - provided_keys): raise ValueError(f"Missing value from user configuration: '{key}'; " "against {required_keys}") for key in (provided_keys - required_keys): raise ValueError(f"Unknown value in user configuration: '{key}'; " f"against {required_keys}") for account in provided_accounts.values(): if not isinstance(account, str): raise ValueError(f"Invalid value for account or currency: '{account}'") def idify(string: str) -> str: """Replace characters objectionable for a filename with underscores. Args: string: Any string. Returns: The input string, with offending characters replaced. """ for sfrom, sto in [(r"[ \(\)]+", "_"), (r"_*\._*", ".")]: string = re.sub(sfrom, sto, string) string = string.strip("_") return string beangulp-0.2.0/beangulp/utils_test.py000066400000000000000000000076721474340320100176660ustar00rootroot00000000000000from decimal import Decimal from os import path import datetime import logging import os import types import unittest from unittest import mock from shutil import rmtree from tempfile import mkdtemp from beangulp import utils class TestWalk(unittest.TestCase): def setUp(self): self.temp = mkdtemp() def tearDown(self): rmtree(self.temp) def test_walk_empty(self): entries = utils.walk([self.temp]) self.assertListEqual(list(entries), []) def test_walk_simple(self): filenames = [path.join(self.temp, name) for name in ('z', 'a', 'b')] for filename in filenames: with open(filename, 'w'): pass entries = utils.walk([self.temp]) self.assertListEqual(list(entries), sorted(filenames)) entries = utils.walk(filenames) self.assertListEqual(list(entries), filenames) def test_walk_subdir(self): os.mkdir(path.join(self.temp, 'dir')) filenames = [path.join(self.temp, 'dir', name) for name in ('a', 'b')] for filename in filenames: with open(filename, 'w'): pass entries = utils.walk([self.temp]) self.assertListEqual(list(entries), filenames) def test_walk_mixed(self): os.mkdir(path.join(self.temp, 'dir')) files = ['c', ('dir', 'a'), ('dir', 'b')] filenames = [path.join(self.temp, *p) for p in files] for filename in filenames: with open(filename, 'w'): pass entries = utils.walk([self.temp]) self.assertListEqual(list(entries), filenames) class TestUtils(unittest.TestCase): def test_getmdate(self): self.assertIsInstance(utils.getmdate(__file__), datetime.date) def test_logger(self): logger = utils.logger() self.assertIsInstance(logger, types.FunctionType) logger = utils.logger(logging.INFO, err=True) self.assertIsInstance(logger, types.FunctionType) def test_sha1sum(self): self.assertRegex(utils.sha1sum(__file__), '[a-f0-9]+') def test_is_mimetype(self): self.assertTrue(utils.is_mimetype(__file__, {'text/x-python'})) self.assertTrue(utils.is_mimetype(__file__, 'text/x-python')) def test_search(self): self.assertTrue(utils.search_file_regexp(__file__, 'def test_search', encoding='utf8')) self.assertFalse(utils.search_file_regexp(__file__, '^$', encoding='utf8')) def test_parse_amount(self): self.assertEqual(Decimal('-1045.67'), utils.parse_amount('(1,045.67)')) def test_validate(self): utils.validate_accounts( {'cash': 'Cash account', 'position': 'Cash account'}, {'cash': 'Assets:US:Cash', 'position': 'Assets:Investment'}) # Missing values. with self.assertRaises(ValueError): utils.validate_accounts( {'cash': 'Cash account', 'position': 'Cash account'}, {'position': 'Assets:Investment'}) # Unknown values. with self.assertRaises(ValueError): utils.validate_accounts( {'cash': 'Cash account'}, {'cash': 'Assets:US:Cash', 'position': 'Assets:Investment'}) # Invalid values. with self.assertRaises(ValueError): utils.validate_accounts( {'cash': 'Cash account'}, {'cash': 42}) def test_idify(self): self.assertEqual( "A_great_movie_for_us.mp4", utils.idify(" A great movie (for us) .mp4 ") ) self.assertEqual("A____B.pdf", utils.idify("A____B_._pdf")) class TestDefDictWithKey(unittest.TestCase): def test_defdict_with_key(self): factory = mock.MagicMock() testdict = utils.DefaultDictWithKey(factory) testdict["a"] testdict["b"] self.assertEqual(2, len(factory.mock_calls)) self.assertEqual(("a",), factory.mock_calls[0][1]) self.assertEqual(("b",), factory.mock_calls[1][1]) beangulp-0.2.0/etc/000077500000000000000000000000001474340320100140575ustar00rootroot00000000000000beangulp-0.2.0/etc/env000077500000000000000000000000521474340320100145720ustar00rootroot00000000000000#!/bin/sh PYTHONPATH=$PYTHONPATH:$PROJDIR beangulp-0.2.0/examples/000077500000000000000000000000001474340320100151225ustar00rootroot00000000000000beangulp-0.2.0/examples/Downloads/000077500000000000000000000000001474340320100170545ustar00rootroot00000000000000beangulp-0.2.0/examples/Downloads/Statement.pdf000066400000000000000000000344061474340320100215220ustar00rootroot00000000000000%PDF-1.4 %äüöß 2 0 obj <> stream xSMO0 Wv⤕Hl&qT‸8qmmJMgbEYlqcq{o;+v]LB\UgL?ݵHM:mЗCp_]_FVL3^&D0t5Df vЃ Y=QIt*WH|(jCҵTRd&|KMFwxMY@I(,GrN}wU#>`Ѡu^œV@uniлf;ܨNԥ"Yɡ^G2u(}vbs7Dq)U_ݷ56K=E)pYt.cZwYz\x-D$.uMʼn-+[?`o-aU|ћ endstream endobj 3 0 obj 385 endobj 4 0 obj <> /Length 8 /Filter/FlateDecode >> stream x endstream endobj 5 0 obj <> endobj 7 0 obj <> stream x{ tSǵ̜su,:BmYe,}|l؀06Md $MqĄ M!64-m)ݤ}mzͧn KJߞ#O6kzkc̞={{{ӓ; IW~1BԿsZX"-sܖm#i2<7@J*Aa@w+lz7kA}꧶)^oPows,.m\3ţP߇u|ljze-"T{O/^:NS(UjVO1FbSiL^fo/W h\t1g|.(T8zG/Q$Єh̍OK0F_FttQt=w,כFѝ0_`*c}Bw[IH ܆dpk$9V(m!£c8O<]q0=4v,_\GB F:Ə3_;zd=XjTs\}*C[ۇaS8#>Ѿm}x5-WVIbMuՊʊE‚ܜlogەj1VV)=MQ!ٞP=} j&&De2fJ (>F)%(kQuAb/5x {}'<]BlA0-WPqИ: pTh5m6ZJAMA>:(?sE,$qYTzmw668UO܄V,c1RCGs\_7軵3AYqv@< 1ocƘrmi1{?#g᝛1}IQ ;;f}g6{3{Vo p'̓X]1>:W$'3#&a0+ncM5#dvӉ PͬLyIWDiR,\6[6tXO#`_lf3mT>5@L+V 1.nBr%DdMBP>hs8?ΘԗQ٢苂FdSMtX#:.n1'{ Bl!1˳y\?["8 @m+g;bsVڐtǤ.PpsH7sO-<-;+I4Pvcl<0ʫ:BBjǔ^|yZ_-tb'Za|B`CobQsZZ⦠U2twWA>f!`衢B -51^#FFQYR:=.ϰtnT<eUMbBnh^Paƚkת5ZjfU 'W5aW?]Ϟ>Xİ<{VZvֳj`ֳZr,j-z}J ݝC&y`2Zu6 : VXBI+PNmPQ%fVVF 8 $ppl'8zRAt F#H>8="H#Di<1k)6WP,pA|ϩrF p: V³)YBYz92Q9BQ9LAu Dga_i`_B]BsJ!!5Mؤ)pu+%R Q,!!gWbIV3Jł߲`˅Źg:C % &Q%e # Yr۳i l20b-Q\d lB`'b DN{#&8_H}y.-Al%f0/ś~?<?WJKK_P;h߾{ѰdL o`cM2^;κϪ!R<] MENRiCWT5lvQB0P DG"Oˋ`zpq˲ qi,+XIfON!,K!VK& Q/5&dcE}f<;)NI)ݱr׮|kAEݤQB@n$ 8TRƐazUj$&ƔF5j5CntJ]Qau0FU3*GzWc*MidgT ؊TucpNwSe$_(SlM \^F`acN[Vex3g1|gΜ:TJ?bJӮl˯ɀ dF3.ÚA8UC*{IaR]{Ny=Ze[Pkb0;3M)Q)RIzYF>2 ׊Y-+!F"&]x Ug0 ¦vk6$ǰJ5T֛41_ _T|o0PK 0WU-}C0)5'5p:O8; &eD! P6c *-݂c&EA)Ƃ ]H ۩XͥN(: ԼWC 5@V5r #<.( MSTGUD=Л &rbG&&{"= 'E mSeOU^g\RH@tM͞i'l ՒB -<˲=w=k?k;3 z\.,{YQ3dз'|o9`/UUq.y MC+W Q kB?jו%&nܖJ١w {(G$l$&ڈIBљވ1 1Jה5+*f*po*0˛-9T.MJ('ghv28=k 6EXm]F,LAb e0@$Hoʈ.Ƴl-xB€="6+S@~rSi4\#7 ̲UERyE*|oƊ RW\Qcl796fk,NOx)d0,\vayys#$O͇XVx.dڷĦbk}V~%^*TQ/bqϺd-Қg\s.\KkޥPǐk] !KG4 =XB1^Lu/ LpI`%+7ܐ=;ztԴ'~1Q>+ӜY[cЫPi٫fZcGo9Y"{):CCHQt-M:JۢݤejQv!vOlӫ:'"yڲCQOF~1K׾?6mM[|}_nir3ّsG+wBz[~5}oc'Z!ȁn*Ib'sZ:'Ѥ43aKB,%bxF`jIR+q=W=`"F"4 p˲KR֠cب0xmλk߫\U-䡂ꞺzNc1h9{txi~ >z143IԩT_꣩*;j](3/3pWR8t PUaXo<0ɉ9_Ðuұ'pu"Ӽ *ìv2Y_/v@O.\b}?h/MoTEiK6X`㝭wn $CMFc|tM!}U*BGYP|"&T1|1ILz>dA253#&*$wvH#n3# Vi'=$gD /2 Vv~ٽF@rx"T2ƏCf!ߑ/Ls$ʛ:f;(V!;@8RgY &!ԆمX݌R!e.R|BJM#Rh23J4i"$7MuJXge? Fdƌ@± ssC!+%x]ؤ#,Jo\_ ;3aksv)GOITDء8l!%ߌﲏ5+C$#[shRl=gKքS,Y"4Ox O96-֕wWGKVϛޑʺFOf˚WV+fʙw<¬IkZ` Upj ~q/ŞGz_Ň|J I`5X KȚ˺5PeyùeeLsi~ZpSR@8h42&Z ]bÔ\c)Bφ)B2`l!j{v9.$~rLXCKͪ-+ R\:&:ǜdpX NAp3jNDI$O"+K3Ωby?SJ~+`k8^)bNLoFv&ɭce$q"WaDhbKS$F*M["LL}v-ӊ͙]ikT><͹"24/eqڷ! 7L5RԊ{$Am0kNc'^>=vx/BI HX7.X4*FąS\TrF@*/dKI ^{\7HjJnMYk/=dG5#5U7S:.9c=ày>Δ>\JXFOeyyYU%f~P}*T8U1_4T* "exٲ)X,e}a*Hd!qhFH *3"VIVJOQdI`kP(A`̿?:0]>X1A{D[ }@Oq|Yf2jW0>5_A_~+ kxaҡ@xe^Ǟyu~ *{ +U=*L0֔}[ K I]fS-D3c k)X=f2 ^^jp|ezdrHP0Vx*YVVexo>ޙ7<^˦){0.3{zӐ \i,JȻT(tɩt_$N78a'Ex56ܚ:*~zKϵyg«6-sʩoۍGw}+B=uD*J3'66FmY =R8 שf%˖J?(%\[ʰ%JWb-0vΕr1#!m.~;J.r.ɥ6A6Ჭlz^FkoۯEvb9۶+6r(FyaV ­`x9^1bY>_Ej|L4fI)-"V/oD|^|A$O7l/WŷD月OL(c"%bY ⟋ ">&>'}"). "I&ǽ*2O/I8,zg)PVį"/,Hg΅"iCRyDn[7Oh_olo;'[W!现tE\lOLm QN_֞ո#C.w<ґXk5H*5o9qVRi,7u4zz]Τ0"~Av'i[^=QqcױqqkǢCuف;w`pA;24`b8N r9908j̢_r|ʁk:^.vO8ƀӞ/E{qA8pR:<4XNo,?_q t>lCrˁaؿӈ9H/9Hy~ aSD+&Y2s43JLu3FXTe5v=RcD5"Sb/] -W|]`7iM=7uk=f$B-q n nU? =udkp{V,dT?U*8Vgi"N/C^8U(e.-&>`%+lmwfm,A:53ifj]+֐knBB9.yQ;r)]M7pyawIvC6v|+6hOkA ݽ'w` j9'_["9/EҠƘoM| 䕜{{֮,wm 5ܲ閆W:X6YNZXXZ|-o*kB벸H.|ąIus͹#2).3 Gؾtb3^j"\y`(⏝B1]ԝ\-'ẖr3/~7bg؋R>Q 4ꤊhT {%j#G Q҅`܄wy7KkrIVwq7Qܔ- B cX;Ì-EYH`o^Ke㠛ryhSF翥۔|Z "z@M~EӻHyȣ}Olz/;e|Y\tLc9{X(.*&bdgj2cR1,p3 =zQzӿ=,>d fO|@)?获a+.[|p Y)jRBJa=ӟ MHD ~/N!Z2Y$IٷSe/Xe{%i'5O= {S|;ds{o|6l04CJrw*Vݯ"u,VG~ _?3?*_~A/ 9C`gYa kxBzپ4yY '^\{!)_^ Ƥ)dj[lgeFWvo{b,8UJ_ ]ކhUm޼{<7;QՂ3|Z#]9vQ9#QK,+p7Ns9(LE~qShqrf86>[%d"''=9@Edymm%|Ge[p`΀;.u%+jSUʠh6h<o)\.WK\Q85V)Ps,9z2>ʴ:à` 6B=cHR8JŘTKbUtT^O8+*#|KgnǙXv|tQ/K7e֏!G=;ǹQ:" v"35FU B)R1F3ŤpxKŗ+~ތ4=詾eܸoSwxkbÏ?x_zkGxrf?ZWlB[ѝ$bhbi+/@BImSH+; 6I #ڰ6*pYhqY&AdzUʆ 50LYPclȿ(6.BgDloAJĔ[+FܒpQK>2֍ZX~ч͹SE5)&k*!zSHYDT[\_+%0^'n$'q&ڃ`ΰC"7qS1҃Q4hdrC=jvc ƿ̨_f_ߕ[*,|mi./]ޮ5=m,KyPђ?ՂT=.}n !e__j-eʻ0C?Of={ƍ4,)ikǴgy\TNJSu9 gX6ܦCvmiyL{L_^A+WV\^!e d:G,_e jmft~Ǩ8%̄"E'd7' l'ԡ.BZq SOn/iycKM˹] swvn'WofNoy|kyx?:e4uqoQp_jlD.r`wxw"oeye ,TY'dVxeEH*l|{#O\?$ Zx# KA ܿ" +3P;TEC9B,BMD||^ mnyGldJ^& CПҭJ'|OF` wRA*s-R~hW+ET yQ&=/Ǎ yy?,Y7ZgRE%(@̰Z-#K/o&5a&ah< 3.v%ahLJAO$a2XV;$B\(IZæk=W$Dbc|<#mI#1%aR$̠2FJ,LJg%ad%a%U(}! Q:N֢ NuVn=Sgw=9exZΤ\hB}ªBn6A&&'w W[+L }dɭ[G6NM 'G7Y7?8:08)x},/,*/ ^o8 Fedjzp#BGB!7=8:-:}@<6= CmH?}TKizp砰ozzpjltxzz|E k׮¾$q?mS#[Fa۷€F`;'hnZ((g[&_()?48CmplrK`֑@u6K9h%5M i$ K@ŐA"64 >hBm:)6 ;徔zգFV^vD4P $dsn>oC[F~Z7ȘK{nA;`c<0S&@?iehZr[! ~bO_HB[d.2̻(6Ta'ŴQ8S˼<pR' 6O[P+v\#emrmA fI2(OM 8(y &t^(՚ШlTB;ncB6֚2rćj角8@ ey#nؖ$YDh}uV.̪1+uX\+x#?]|2Y^{ϼ aZ хS Coc+\f1F PiL sEss3s.ϩf{; v}]gi>=I<  <Ɯxhsc9c?+=76>h u  `51Dfl Ŷ*)y7л;/:7u=tgĊ7tƢ]-$ gi;k?lw OM)m24h}0BF[2R{7 endstream endobj 8 0 obj 11603 endobj 9 0 obj <> endobj 10 0 obj <> stream x]n0 6jTv6Рӌ}i!MMվ./A٫Z}g> {,S >CZڷGTk-*4V5^1H(SiQdߞI˥?+K/~C|`Ns# !7 ?oA?o(G!-&&-,tv-[p;ſ &3s|⟐,Y`sÉ/}]~ϟR9?R<> endobj 12 0 obj <> endobj 13 0 obj <> /ExtGState<> /ProcSet[/PDF/Text/ImageC/ImageI/ImageB] >> endobj 1 0 obj <>/Contents 2 0 R>> endobj 6 0 obj <> endobj 14 0 obj <> endobj 15 0 obj < /Producer /CreationDate(D:20160321005914-04'00')>> endobj xref 0 16 0000000000 65535 f 0000013560 00000 n 0000000019 00000 n 0000000475 00000 n 0000000495 00000 n 0000000674 00000 n 0000013703 00000 n 0000000714 00000 n 0000012402 00000 n 0000012424 00000 n 0000012619 00000 n 0000013083 00000 n 0000013402 00000 n 0000013435 00000 n 0000013802 00000 n 0000013899 00000 n trailer < <2FC3F9B6EC5091B690969A624FCC56BE> ] /DocChecksum /FA923EC16E7805D3ACAB850086F664C4 >> startxref 14074 %%EOF beangulp-0.2.0/examples/Downloads/UTrade20160215.csv000066400000000000000000000015141474340320100215770ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-11-17,SELL,35166597,SOLD +MSFX 41 @84.22 (LOT 93.91),7.95,3445.07,4522.01 2015-12-03,DIV,48233019,ORDINARY DIVIDEND~CSKO,0,9.01,4531.02 2015-12-14,DIV,93099952,ORDINARY DIVIDEND~HOOL,0,7.00,4538.02 2015-12-18,DIV,68604400,ORDINARY DIVIDEND~HOOL,0,6.96,4544.98 2015-12-21,DIV,19159651,ORDINARY DIVIDEND~CSKO,0,18.40,4563.38 2015-12-29,BUY,30632545,BOUGHT +BAPL 71 @58.42,7.95,-4155.77,407.61 2016-01-05,DIV,69390977,ORDINARY DIVIDEND~BAPL,0,19.04,426.65 2016-01-18,DIV,50183865,ORDINARY DIVIDEND~MSFX,0,8.25,434.90 2016-01-28,DIV,90876232,ORDINARY DIVIDEND~HOOL,0,8.26,443.16 2016-02-02,DIV,78355964,ORDINARY DIVIDEND~MSFX,0,8.04,451.20 2016-02-05,SELL,65828025,SOLD +CSKO 81 @51.83 (LOT 63.85),7.95,4190.28,4641.48 2016-02-07,DIV,18110667,ORDINARY DIVIDEND~CSKO,0,24.41,4665.89 beangulp-0.2.0/examples/Downloads/ofxdownload.ofx000066400000000000000000000064701474340320100221250ustar00rootroot00000000000000OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:NONE ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE 0INFOLogin successful20140112083600.212[-7:MST]ENGAMEX3101FMPWeb310120140112083600exampleuser00INFOUSD379700001111222falsedownload90Days379700001111222trueBe308f58246398a74c52504a8b06d5f0520140112050000.000[-7:MST]20140112050000.000[-7:MST]DEBIT20131201000000.000[-7:MST]-9.99320133350353869664320133350353869664SPOTIFY USA 287701309012600720879 WWW.SPOTIFY.COMDEBIT20131202000000.000[-7:MST]-61.71320133360368356592320133360368356592GOAT TOWN 1200000549NEW YORK 071000163 2126873641DEBIT20131202000000.000[-7:MST]-17.75320133360368356593320133360368356593AMC VILLAGE 7 #2110 NEW YORK 12010365699 212-982-2116DEBIT20131205000000.000[-7:MST]-42.4320133390419136515320133390419136515CAFE MOGADOR 0048 NEW YORK 969881 212-677-2226DEBIT20131209000000.000[-7:MST]-34.38320133430477621377320133430477621377UNION MARKET - HOUSNEW YORK 15827 GROCERY STOREDEBIT20131209000000.000[-7:MST]-13.5320133430477621378320133430477621378SUNSHINE CINEMA 5429NEW YORK 208001222 2122607289DEBIT20131211000000.000[-7:MST]-21.96320133450009270816320133450009270816UNION MARKET - HOUSNEW YORK 13775 GROCERY STOREDEBIT20131211000000.000[-7:MST]-14.47320133450009270817320133450009270817WHOLEFDS HOU 10236 02124201320042002720272124201320DEBIT20131214000000.000[-7:MST]-56.43320133480059384557320133480059384557MACY'S #003 HERALD SNEW YORK 00307963916 MACY'SDEBIT20131215000000.000[-7:MST]-13320133490073491495320133490073491495WHOLEFDS HOU 10236 02124201320042402720282124201320DEBIT20131216000000.000[-7:MST]-60.23320133500088330906320133500088330906PAPYRUS #2302 000002NEW YORK 14252302002 2162527300DEBIT20131216000000.000[-7:MST]-7320133500088330907320133500088330907LES CAFES 400 LAFAYENEW YORK 10156320131 2157761076-2356.3820131218050000.000[-7:MST]falsefalsefalse beangulp-0.2.0/examples/README.md000066400000000000000000000103261474340320100164030ustar00rootroot00000000000000# Beangulp Example This directory contains an example for how to organize yourself and your files to automate your Beancount ledger update. See http://furius.ca/beancount/doc/beangulp for a fuller documentation and discussion of these files. ## Example Files Organization - Sophisticated There are five directories demonstrated here: * `ledger`: This is a directory, typically a repository under source control, containing your Beancount ledger files. * `documents`: This is a directory containing archived imported files. The directory hierarchy mirrors the structure of the accounts in the ledger and is constructed by the "archive" command which can also automatically date and rename the files and place them in the correct location after you have finished updating your Beancount ledger. * `importers`: This is a directory, typically a repository under source control, containing your custom importers implementation. Note that in the most general case this does not include examples of your downloaded files nor any personal account-specific information, because you may want to share your importers with others. * `tests`: This directory contains examples of real downloaded files from your institutions which will serve as regression tests. Next to each downloaded file is a `.beancount` golden file with the correct contents extracted by the importer. Those should be generated by the "generate" command from the importer and eyeballed for correctness. Running the "test" after a change of your importer code will verify the importer's updated output agains the existing golden files. This is a really fast way to add some testing and regression detection around your custom importer code. * `Downloads`: This is the location where your browser might drop the files dowloaded from your statement sources. ## Example Files Organization - Simpler Note that you could further simplify and merge some of these together for your convenience. The example above shows that in the most general case you could store all of these things separately. In fact, the first four directories could all be stored to a single repository, if you wanted to keep things really simple. We recommend you start that way, especially if all the information is overwhelming. Here's an example for how you could start with all your files in one directory: ledger/ ├── import.py ├── ledger.beancount ├── importers │ ├── acme │ │ ├── acmebank1.pdf │ │ ├── acmebank1.pdf.beancount │ │ └── acme.py │ └── utrade │ ├── UTrade20140713.csv │ ├── UTrade20140713.csv.beancount │ ├── UTrade20150225.csv │ ├── UTrade20150225.csv.beancount │ ├── UTrade20150720.csv │ ├── UTrade20150720.csv.beancount │ └── utrade.py └── documents ├── Assets │ └── US │ ├── AcmeBank │ └── UTrade │ └── ... ├── Expenses ├── Income └── Liabilities └── US └── CreditCard └── ... ## How to run this example The below steps have been tested with Linux and Windows Subsystem for Linux (WSL1/2). Clone the example from beangulp and cd into the folder: ```bash git clone git@github.com:beancount/beangulp.git cd beangulp/examples ``` Install beangulp and beancount in a `.venv`. ```bash apt-get install python3-venv # required for virtual env python3 -m venv .venv # create in subfolder called ".venv" source ./.venv/bin/activate pip install beangulp beancount ``` At this stage, make sure you are not installing fava (`pip install beangulp beancount fava`) because this still has beancount v2.3.6 pinned (if you need fava, install in a different venv). Also, if you want to run the example folder fully, including pdf2text extraction, install the following dependencies for pdftotext: ```bash apt-get install poppler-utils ``` Now run beancount with the beangulp importer: ```bash python import.py extract ./Downloads > tmp.beancount ```beangulp-0.2.0/examples/documents/000077500000000000000000000000001474340320100171235ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/.keep000066400000000000000000000000001474340320100200360ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/000077500000000000000000000000001474340320100203655ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/.keep000066400000000000000000000000001474340320100213000ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/US/000077500000000000000000000000001474340320100207145ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/US/AcmeBank/000077500000000000000000000000001474340320100223555ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/US/AcmeBank/.keep000066400000000000000000000000001474340320100232700ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/US/UTrade/000077500000000000000000000000001474340320100221005ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Assets/US/UTrade/2014-07-13.utrade.UTrade20140713.csv000066400000000000000000000017461474340320100271500ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-04-14,BUY,14167001,BOUGHT +CSKO 50 @98.35,7.95,-4925.45,25674.63 2014-05-08,BUY,12040838,BOUGHT +HOOL 121 @79.11,7.95,-9580.26,16094.37 2014-05-11,BUY,41579908,BOUGHT +MSFX 104 @64.39,7.95,-6704.51,9389.86 2014-05-22,DIV,54857517,ORDINARY DIVIDEND~HOOL,0,28.56,9418.42 2014-05-23,XFER,27634682,CLIENT REQUESTED ELECTRONIC FUNDING,0,7148.74,16567.16 2014-05-25,DIV,31749124,ORDINARY DIVIDEND~CSKO,0,9.63,16576.79 2014-05-28,BUY,83788120,BOUGHT +HOOL 92 @52.10,7.95,-4801.15,11775.64 2014-05-29,DIV,97871874,ORDINARY DIVIDEND~HOOL,0,7.49,11783.13 2014-06-05,DIV,85665025,ORDINARY DIVIDEND~CSKO,0,16.32,11799.45 2014-06-14,DIV,31730597,ORDINARY DIVIDEND~HOOL,0,15.60,11815.05 2014-06-21,BUY,22346704,BOUGHT +MSFX 101 @78.00,7.95,-7885.95,3929.10 2014-06-23,DIV,36811051,ORDINARY DIVIDEND~HOOL,0,8.70,3937.80 2014-07-01,DIV,30631356,ORDINARY DIVIDEND~MSFX,0,9.39,3947.19 2014-07-12,DIV,33403638,ORDINARY DIVIDEND~MSFX,0,16.64,3963.83 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2014-10-01.utrade.UTrade20141001.csv000066400000000000000000000006651474340320100271250ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-07-12,DIV,33403638,ORDINARY DIVIDEND~MSFX,0,16.64,3963.83 2014-07-28,DIV,23571586,ORDINARY DIVIDEND~CSKO,0,15.60,3979.43 2014-08-07,BUY,90404110,BOUGHT +CSKO 48 @52.93,7.95,-2548.59,1430.84 2014-08-22,DIV,70106713,ORDINARY DIVIDEND~HOOL,0,22.58,1453.42 2014-08-26,DIV,18874360,ORDINARY DIVIDEND~CSKO,0,8.46,1461.88 2014-09-24,DIV,18146488,ORDINARY DIVIDEND~CSKO,0,7.36,1469.24 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2014-12-14.utrade.UTrade20141214.csv000066400000000000000000000012441474340320100271330ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-09-24,DIV,18146488,ORDINARY DIVIDEND~CSKO,0,7.36,1469.24 2014-10-13,DIV,42072631,ORDINARY DIVIDEND~CSKO,0,11.39,1480.63 2014-10-14,DIV,45612217,ORDINARY DIVIDEND~HOOL,0,17.45,1498.08 2014-10-22,SELL,84625538,SOLD +CSKO 22 @88.13 (LOT 98.35),7.95,1930.91,3428.99 2014-11-09,DIV,65176118,ORDINARY DIVIDEND~HOOL,0,7.01,3436.00 2014-11-27,SELL,92392307,SOLD +CSKO 93 @33.31 (LOT 32.59),7.95,3089.88,6525.88 2014-12-01,DIV,10447525,ORDINARY DIVIDEND~CSKO,0,6.82,6532.70 2014-12-02,BUY,74330963,BOUGHT +HOOL 55 @66.61,7.95,-3671.50,2861.20 2014-12-13,SELL,79580321,SOLD +HOOL 50 @39.13 (LOT 42.33),7.95,1948.55,4809.75 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2015-02-25.utrade.UTrade20150225.csv000066400000000000000000000017561474340320100271470ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-11-27,SELL,92392307,SOLD +CSKO 93 @33.31 (LOT 32.59),7.95,3089.88,6525.88 2014-12-01,DIV,10447525,ORDINARY DIVIDEND~CSKO,0,6.82,6532.70 2014-12-02,BUY,74330963,BOUGHT +HOOL 55 @66.61,7.95,-3671.50,2861.20 2014-12-13,SELL,79580321,SOLD +HOOL 50 @39.13 (LOT 42.33),7.95,1948.55,4809.75 2014-12-20,DIV,60292945,ORDINARY DIVIDEND~CSKO,0,5.17,4814.92 2014-12-22,DIV,98071623,ORDINARY DIVIDEND~HOOL,0,19.97,4834.89 2014-12-31,BUY,94789086,BOUGHT +MSFX 87 @50.34,7.95,-4387.53,447.36 2015-01-05,DIV,89525843,ORDINARY DIVIDEND~CSKO,0,23.67,471.03 2015-01-17,XFER,92120597,CLIENT REQUESTED ELECTRONIC FUNDING,0,4876.00,5347.03 2015-01-19,BUY,76302399,BOUGHT +HOOL 40 @99.15,7.95,-3973.95,1373.08 2015-02-02,DIV,57564142,ORDINARY DIVIDEND~HOOL,0,7.98,1381.06 2015-02-06,DIV,90692243,ORDINARY DIVIDEND~BAPL,0,12.27,1393.33 2015-02-20,DIV,34606455,ORDINARY DIVIDEND~BAPL,0,14.83,1408.16 2015-02-22,DIV,16360724,ORDINARY DIVIDEND~BAPL,0,24.24,1432.40 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2015-05-05.utrade.UTrade20150505.csv000066400000000000000000000014561474340320100271460ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-02-06,DIV,90692243,ORDINARY DIVIDEND~BAPL,0,12.27,1393.33 2015-02-20,DIV,34606455,ORDINARY DIVIDEND~BAPL,0,14.83,1408.16 2015-02-22,DIV,16360724,ORDINARY DIVIDEND~BAPL,0,24.24,1432.40 2015-02-26,DIV,39293066,ORDINARY DIVIDEND~HOOL,0,10.71,1443.11 2015-03-06,XFER,95397432,CLIENT REQUESTED ELECTRONIC FUNDING,0,5393.93,6837.04 2015-03-20,SELL,47069288,SOLD +BAPL 107 @42.87 (LOT 38.83),7.95,4579.14,11416.18 2015-03-24,BUY,91955231,BOUGHT +BAPL 89 @56.05,7.95,-4996.40,6419.78 2015-03-30,XFER,68299938,CLIENT REQUESTED ELECTRONIC FUNDING,0,3944.15,10363.93 2015-04-01,DIV,11669685,ORDINARY DIVIDEND~HOOL,0,4.69,10368.62 2015-04-02,BUY,35067825,BOUGHT +CSKO 149 @63.85,7.95,-9521.60,847.02 2015-04-14,DIV,26605450,ORDINARY DIVIDEND~BAPL,0,9.95,856.97 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2015-07-20.utrade.UTrade20150720.csv000066400000000000000000000015421474340320100271400ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-05-17,DIV,57567122,ORDINARY DIVIDEND~HOOL,0,15.96,872.93 2015-05-18,SELL,98562913,SOLD +BAPL 146 @32.01 (LOT 26.15),7.95,4665.51,5538.44 2015-05-19,DIV,45119594,ORDINARY DIVIDEND~HOOL,0,13.91,5552.35 2015-05-26,SELL,53367649,SOLD +CSKO 89 @66.30 (LOT 63.85),7.95,5892.75,11445.10 2015-06-07,BUY,56695140,BOUGHT +HOOL 777 @5.16,7.95,-4017.27,7427.83 2015-06-19,DIV,17207133,ORDINARY DIVIDEND~BAPL,0,12.79,7440.62 2015-06-20,DIV,50840576,ORDINARY DIVIDEND~BAPL,0,13.89,7454.51 2015-06-25,DIV,88647893,ORDINARY DIVIDEND~BAPL,0,10.13,7464.64 2015-06-29,BUY,22740217,BOUGHT +CSKO 54 @64.93,7.95,-3514.17,3950.47 2015-07-02,DIV,27217440,ORDINARY DIVIDEND~HOOL,0,26.19,3976.66 2015-07-18,DIV,64873832,ORDINARY DIVIDEND~BAPL,0,9.80,3986.46 2015-07-19,BUY,51600680,BOUGHT +HOOL 31 @97.08,7.95,-3017.43,969.03 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2015-09-24.utrade.UTrade20150924.csv000066400000000000000000000011701474340320100271510ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-06-29,BUY,22740217,BOUGHT +CSKO 54 @64.93,7.95,-3514.17,3950.47 2015-07-02,DIV,27217440,ORDINARY DIVIDEND~HOOL,0,26.19,3976.66 2015-07-18,DIV,64873832,ORDINARY DIVIDEND~BAPL,0,9.80,3986.46 2015-07-19,BUY,51600680,BOUGHT +HOOL 31 @97.08,7.95,-3017.43,969.03 2015-07-26,DIV,75756901,ORDINARY DIVIDEND~HOOL,0,14.45,983.48 2015-08-18,DIV,28729581,ORDINARY DIVIDEND~CSKO,0,8.15,991.63 2015-08-31,DIV,97130478,ORDINARY DIVIDEND~BAPL,0,11.21,1002.84 2015-09-13,DIV,74624169,ORDINARY DIVIDEND~CSKO,0,12.47,1015.31 2015-09-19,DIV,86506548,ORDINARY DIVIDEND~MSFX,0,14.07,1029.38 beangulp-0.2.0/examples/documents/Assets/US/UTrade/2015-12-07.utrade.UTrade20151207.csv000066400000000000000000000007771474340320100271530ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-09-13,DIV,74624169,ORDINARY DIVIDEND~CSKO,0,12.47,1015.31 2015-09-19,DIV,86506548,ORDINARY DIVIDEND~MSFX,0,14.07,1029.38 2015-10-21,DIV,71819731,ORDINARY DIVIDEND~HOOL,0,7.98,1037.36 2015-10-26,DIV,99810091,ORDINARY DIVIDEND~HOOL,0,16.83,1054.19 2015-11-15,DIV,84191955,ORDINARY DIVIDEND~BAPL,0,22.75,1076.94 2015-11-17,SELL,35166597,SOLD +MSFX 41 @84.22 (LOT 93.91),7.95,3445.07,4522.01 2015-12-03,DIV,48233019,ORDINARY DIVIDEND~CSKO,0,9.01,4531.02 beangulp-0.2.0/examples/documents/Expenses/000077500000000000000000000000001474340320100207155ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Expenses/.keep000066400000000000000000000000001474340320100216300ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Income/000077500000000000000000000000001474340320100203355ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Income/.keep000066400000000000000000000000001474340320100212500ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Liabilities/000077500000000000000000000000001474340320100213555ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Liabilities/.keep000066400000000000000000000000001474340320100222700ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Liabilities/US/000077500000000000000000000000001474340320100217045ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Liabilities/US/CreditCard/000077500000000000000000000000001474340320100237105ustar00rootroot00000000000000beangulp-0.2.0/examples/documents/Liabilities/US/CreditCard/2013-12-03.bofa.ofx000066400000000000000000000144021474340320100263620ustar00rootroot00000000000000OFXHEADER:100 DATA:OFXSGML VERSION:102 SECURITY:NONE ENCODING:USASCII CHARSET:1252 COMPRESSION:NONE OLDFILEUID:NONE NEWFILEUID:NONE 0INFOLogin successful20140112083600.212[-7:MST]ENGAMEX3101FMPWeb310120140112083600exampleuser00INFOUSD379700001111222falsedownload90Days379700001111222trueBe308f58246398a74c52504a8b06d5f0520140112050000.000[-7:MST]20140112050000.000[-7:MST]DEBIT20131121000000.000[-7:MST]-29320133250213757584320133250213757584JEFFREY'S 0252 NEW YORK 0000000681 646-429-8383DEBIT20131122000000.000[-7:MST]-13.93320133260227320537320133260227320537WHOLEFDS HOU 10236 02124201320042102720272124201320DEBIT20131123000000.000[-7:MST]-54.99320133270242332224320133270242332224AMAZON.COM AMZN.COM/BIV7P27IJ69JX MERCHANDISEDEBIT20131124000000.000[-7:MST]-143.94320133280255184014320133280255184014PRUNE NEW YORK 7101466 RESTAURANTDEBIT20131125000000.000[-7:MST]-28.05320133290268683266320133290268683266TAKAHACHI RESTAURANTNEW YORK 000451990 RESTAURANTDEBIT20131126000000.000[-7:MST]-18.76320133300285014247320133300285014247UNION MARKET - HOUSNEW YORK 47155 GROCERY STOREDEBIT20131129000000.000[-7:MST]-23.18320133330323934847320133330323934847WHOLEFDS HOU 10236 02124201320042802720282124201320DEBIT20131129000000.000[-7:MST]-61.98320133330323934848320133330323934848T-MOBILE RECURNG PMTT-MOBILE1070888371 828422957 98006DEBIT20131201000000.000[-7:MST]-9.99320133350353869664320133350353869664SPOTIFY USA 287701309012600720879 WWW.SPOTIFY.COMDEBIT20131202000000.000[-7:MST]-61.71320133360368356592320133360368356592GOAT TOWN 1200000549NEW YORK 071000163 2126873641DEBIT20131202000000.000[-7:MST]-17.75320133360368356593320133360368356593AMC VILLAGE 7 #2110 NEW YORK 12010365699 212-982-2116DEBIT20131026000000.000[-7:MST]-18.92320132990353492589320132990353492589DUANE READE #14354 0NEW YORK 99999993299 8002892273DEBIT20131030000000.000[-7:MST]-16.59320133030406003327320133030406003327WHOLEFDS HOU 10236 02124201320042902720262124201320DEBIT20131102000000.000[-7:MST]-30.49320133060449360218320133060449360218BARNES & NOBLE 2675 NEW YORK 00001102 BOOK STOREDEBIT20131105000000.000[-7:MST]-18.45320133090491048342320133090491048342CULL AND PISTOL NEW YORK 85133313309 212-255-5672DEBIT20131104000000.000[-7:MST]-10.77320133080475440248320133080475440248UNION MARKET - HOUSNEW YORK 44692 GROCERY STOREDEBIT20131106000000.000[-7:MST]-16.75320133100003771504320133100003771504CAFETASIA - #2 88430NEW YORK 4195 RESTAURANTDEBIT20131109000000.000[-7:MST]-10.72320133130046863746320133130046863746WHOLEFDS HOU 10236 02124201320042802720252124201320DEBIT20131109000000.000[-7:MST]-8.98320133130046863747320133130046863747UNION MARKET - HOUSNEW YORK 108039 GROCERY STOREDEBIT20131108000000.000[-7:MST]-102.1320133120033277045320133120033277045EATALY NY NEW YORK 84988943312 212-229-2560DEBIT20131111000000.000[-7:MST]-172.02320133150072538259320133150072538259AMAZON.COM AMZN.COM/BIYFTFUBNN1O6 MERCHANDISEDEBIT20131111000000.000[-7:MST]-30.09320133150072538260320133150072538260CAFE MOGADOR 0048 NEW YORK 960990 212-677-2226DEBIT20131111000000.000[-7:MST]-15.46320133150072538261320133150072538261UNION MARKET - HOUSNEW YORK 108634 GROCERY STOREDEBIT20131110000000.000[-7:MST]-14.68320133140059335751320133140059335751DUANE READE #14354 0NEW YORK 99999993314 8002892273DEBIT20131110000000.000[-7:MST]-15.9320133140059335752320133140059335752WHOLEFDS HOU 10236 02124201320042902720282124201320DEBIT20131112000000.000[-7:MST]-21.51320133160086762615320133160086762615LOBSTER JOINT 542929NEW YORK 000224167 6468961200DEBIT20131114000000.000[-7:MST]-22.55320133180114671752320133180114671752UNION MARKET - HOUSNEW YORK 12189 GROCERY STORE-2093.0120131203050000.000[-7:MST]falsefalsefalse beangulp-0.2.0/examples/import.py000077500000000000000000000045221474340320100170140ustar00rootroot00000000000000#!/usr/bin/env python3 from importers import acme from importers import csvbank from importers import ofx from importers import utrade from beancount.core import data import beangulp importers = [ utrade.Importer("USD", "Assets:US:UTrade", "Assets:US:UTrade:Cash", "Income:US:UTrade:{}:Dividend", "Income:US:UTrade:{}:Gains", "Expenses:Financial:Fees", "Assets:US:BofA:Checking"), ofx.Importer("379700001111222", "Liabilities:US:CreditCard", "bofa"), acme.Importer("Assets:US:ACMEBank"), csvbank.Importer("Assets:US:CSVBank", "USD"), ] def clean_up_descriptions(extracted_entries): """Example filter function; clean up cruft from narrations. Args: extracted_entries: A list of directives. Returns: A new list of directives with possibly modified payees and narration fields. """ clean_entries = [] for entry in extracted_entries: if isinstance(entry, data.Transaction): if entry.narration and " / " in entry.narration: left_part, _ = entry.narration.split(" / ") entry = entry._replace(narration=left_part) if entry.payee and " / " in entry.payee: left_part, _ = entry.payee.split(" / ") entry = entry._replace(payee=left_part) clean_entries.append(entry) return clean_entries def process_extracted_entries(extracted_entries_list, ledger_entries): """Example filter function; clean up cruft from narrations. Args: extracted_entries_list: A list of (filename, entries) pairs, where 'entries' are the directives extract from 'filename'. ledger_entries: If provided, a list of directives from the existing ledger of the user. This is non-None if the user provided their ledger file as an option. Returns: A possibly different version of extracted_entries_list, a list of (filename, entries), to be printed. """ return [(filename, clean_up_descriptions(entries), account, importer) for filename, entries, account, importer in extracted_entries_list] hooks = [process_extracted_entries] if __name__ == '__main__': ingest = beangulp.Ingest(importers, hooks) ingest() beangulp-0.2.0/examples/importers/000077500000000000000000000000001474340320100171465ustar00rootroot00000000000000beangulp-0.2.0/examples/importers/__init__.py000066400000000000000000000000001474340320100212450ustar00rootroot00000000000000beangulp-0.2.0/examples/importers/acme.py000066400000000000000000000036271474340320100204350ustar00rootroot00000000000000"""Example importer for PDF statements from ACME Bank. This importer identifies the file from its contents and only supports filing, it cannot extract any transactions from the PDF conversion to text. This is common, and I figured I'd provide an example for how this works. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import re import subprocess from dateutil.parser import parse as parse_datetime import beangulp from beangulp import mimetypes from beangulp.cache import cache from beangulp.testing import main @cache def pdf_to_text(filename): """Convert a PDF document to a text equivalent.""" r = subprocess.run(['pdftotext', filename, '-'], stdout=subprocess.PIPE, check=True) return r.stdout.decode() class Importer(beangulp.Importer): """An importer for ACME Bank PDF statements.""" def __init__(self, account_filing): self.account_filing = account_filing def identify(self, filepath): mimetype, encoding = mimetypes.guess_type(filepath) if mimetype != 'application/pdf': return False # Look for some words in the PDF file to figure out if it's a statement # from ACME. The filename they provide (Statement.pdf) isn't useful. text = pdf_to_text(filepath) if text: return re.match('ACME Bank', text) is not None def filename(self, filepath): # Normalize the name to something meaningful. return 'acmebank.pdf' def account(self, filepath): return self.account_filing def date(self, filepath): # Get the actual statement's date from the contents of the file. text = pdf_to_text(filepath) match = re.search('Date: ([^\n]*)', text) if match: return parse_datetime(match.group(1)).date() if __name__ == '__main__': importer = Importer("Assets:US:ACMEBank") main(importer) beangulp-0.2.0/examples/importers/csvbank.py000066400000000000000000000015741474340320100211560ustar00rootroot00000000000000from os import path from beangulp import mimetypes from beangulp.importers import csvbase from beangulp.testing import main class Importer(csvbase.Importer): date = csvbase.Date('Posting Date', '%m/%d/%Y') narration = csvbase.Columns('Description', 'Check or Slip #', sep='; ') amount = csvbase.Amount('Amount') balance = csvbase.Amount('Balance') def identify(self, filepath): mimetype, encoding = mimetypes.guess_type(filepath) if mimetype != 'text/csv': return False with open(filepath) as fd: head = fd.read(1024) return head.startswith('Details,Posting Date,"Description",' 'Amount,Type,Balance,Check or Slip #,') def filename(self, filepath): return 'csvbank.' + path.basename(filepath) if __name__ == '__main__': main(Importer('Assets:US:CSVBank', 'USD')) beangulp-0.2.0/examples/importers/ofx.py000066400000000000000000000250521474340320100203200ustar00rootroot00000000000000"""OFX file format importer for bank and credit card statements. https://en.wikipedia.org/wiki/Open_Financial_Exchange This importer will parse a single account in the OFX file. Instantiate it multiple times with different accounts if it has many accounts. It makes more sense to do it this way so that you can define your importer configuration account by account. Note that this importer is provided as an example and with no guarantees. It's not really super great. On the other hand, I've been using it for more than five years over multiple accounts, so it has been useful to me (it works, by some measure of "works"). If you need a more powerful or compliant OFX importer please consider either writing one or contributing changes. Also, this importer does its own very basic parsing; a better one would probably use (and depend on) the ofxparse module (see https://sites.google.com/site/ofxparse/). """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import datetime import enum import itertools import re from os import path from xml.sax import saxutils import bs4 from beancount.core.number import D from beancount.core import amount from beancount.core import data from beancount.core import flags import beangulp from beangulp import mimetypes class BalanceType(enum.Enum): """Type of Balance directive to be inserted.""" NONE = 0 # Don't insert a Balance directive. DECLARED = 1 # Insert a Balance directive at the declared date. LAST = 2 # Insert a Balance directive at the date following the last # extracted transaction. class Importer(beangulp.Importer): """An importer for Open Financial Exchange files.""" def __init__(self, acctid_regexp, account, basename=None, balance_type=BalanceType.DECLARED): """Create a new importer posting to the given account. Args: account: An account string, the account onto which to post all the amounts parsed. acctid_regexp: A regexp, to match against the tag of the OFX file. basename: An optional string, the name of the new files. balance_type: An enum of type BalanceType. """ self.acctid_regexp = acctid_regexp self.importer_account = account self.basename = basename self.balance_type = balance_type def identify(self, filepath): # Match for a compatible MIME type. if mimetypes.guess_type(filepath, strict=False)[0] not in {'application/x-ofx', 'application/vnd.intu.qbo', 'application/vnd.intu.qfx'}: return False # Match the account id. with open(filepath) as fd: contents = fd.read() return any(re.match(self.acctid_regexp, acctid) for acctid in find_acctids(contents)) def account(self, filepath): """Return the account against which we post transactions.""" return self.importer_account def filename(self, filepath): """Return the optional renamed account filename.""" if self.basename: return self.basename + path.splitext(filepath)[1] def date(self, filepath): """Return the optional renamed account filename.""" with open(filepath) as fd: contents = fd.read() return find_max_date(contents) def extract(self, filepath, existing): """Extract a list of partially complete transactions from the file.""" with open(filepath) as fd: soup = bs4.BeautifulSoup(fd, 'lxml') return extract(soup, filepath, self.acctid_regexp, self.importer_account, flags.FLAG_OKAY, self.balance_type) def extract(soup, filename, acctid_regexp, account, flag, balance_type): """Extract transactions from an OFX file. Args: soup: A BeautifulSoup root node. acctid_regexp: A regular expression string matching the account we're interested in. account: An account string onto which to post the amounts found in the file. flag: A single-character string. balance_type: An enum of type BalanceType. Returns: A sorted list of entries. """ new_entries = [] counter = itertools.count() for acctid, currency, transactions, balance in find_statement_transactions(soup): if not re.match(acctid_regexp, acctid): continue # Create Transaction directives. stmt_entries = [] for stmttrn in transactions: entry = build_transaction(stmttrn, flag, account, currency) entry = entry._replace(meta=data.new_metadata(filename, next(counter))) stmt_entries.append(entry) stmt_entries = data.sorted(stmt_entries) new_entries.extend(stmt_entries) # Create a Balance directive. if balance and balance_type is not BalanceType.NONE: date, number = balance if balance_type is BalanceType.LAST and stmt_entries: date = stmt_entries[-1].date # The Balance assertion occurs at the beginning of the date, so move # it to the following day. date += datetime.timedelta(days=1) meta = data.new_metadata(filename, next(counter)) balance_entry = data.Balance(meta, date, account, amount.Amount(number, currency), None, None) new_entries.append(balance_entry) return data.sorted(new_entries) def parse_ofx_time(date_str): """Parse an OFX time string and return a datetime object. Args: date_str: A string, the date to be parsed. Returns: A datetime.datetime instance. """ if len(date_str) < 14: return datetime.datetime.strptime(date_str[:8], '%Y%m%d') return datetime.datetime.strptime(date_str[:14], '%Y%m%d%H%M%S') def find_acctids(contents): """Find the list of tags. Args: contents: A string, the contents of the OFX file. Returns: A list of strings, the contents of the tags. """ # Match the account id. Don't bother parsing the entire thing as XML, just # match the tag for this purpose. This'll work fine enough. for match in re.finditer('([^<]*)', contents): yield match.group(1) def find_max_date(contents): """Extract the report date from the file.""" soup = bs4.BeautifulSoup(contents, 'lxml') dates = [] for ledgerbal in soup.find_all('ledgerbal'): dtasof = ledgerbal.find('dtasof') dates.append(parse_ofx_time(dtasof.contents[0]).date()) if dates: return max(dates) def find_currency(soup): """Find the first currency in the XML tree. Args: soup: A BeautifulSoup root node. Returns: A string, the first currency found in the file. Returns None if no currency is found. """ for stmtrs in soup.find_all(re.compile('.*stmtrs$')): for currency_node in stmtrs.find_all('curdef'): currency = currency_node.contents[0] if currency is not None: return currency def find_statement_transactions(soup): """Find the statement transaction sections in the file. Args: soup: A BeautifulSoup root node. Yields: A trip of An account id string, A currency string, A list of transaction nodes ( BeautifulSoup tags), and A (date, balance amount) for the . """ # Process STMTTRNRS and CCSTMTTRNRS tags. for stmtrs in soup.find_all(re.compile('.*stmtrs$')): # For each CURDEF tag. for currency_node in stmtrs.find_all('curdef'): currency = currency_node.contents[0].strip() # Extract ACCTID account information. acctid_node = stmtrs.find('acctid') if acctid_node: acctid = next(acctid_node.children).strip() else: acctid = '' # Get the LEDGERBAL node. There appears to be a single one for all # transaction lists. ledgerbal = stmtrs.find('ledgerbal') balance = None if ledgerbal: dtasof = find_child(ledgerbal, 'dtasof', parse_ofx_time).date() balamt = find_child(ledgerbal, 'balamt', D) balance = (dtasof, balamt) # Process transaction lists (regular or credit-card). for tranlist in stmtrs.find_all(re.compile('(|bank|cc)tranlist')): yield acctid, currency, tranlist.find_all('stmttrn'), balance def find_child(node, name, conversion=None): """Find a child under the given node and return its value. Args: node: A bs4.element.Tag. name: A string, the name of the child node. conversion: A callable object used to convert the value to a new data type. Returns: A string, or None. """ child = node.find(name) if not child: return None value = child.contents[0].strip() if conversion: value = conversion(value) return value def build_transaction(stmttrn, flag, account, currency): """Build a single transaction. Args: stmttrn: A bs4.element.Tag. flag: A single-character string. account: An account string, the account to insert. currency: A currency string. Returns: A Transaction instance. """ # Find the date. date = parse_ofx_time(find_child(stmttrn, 'dtposted')).date() # There's no distinct payee. payee = None # Construct a description that represents all the text content in the node. name = find_child(stmttrn, 'name', saxutils.unescape) memo = find_child(stmttrn, 'memo', saxutils.unescape) # Remove memos duplicated from the name. if memo == name: memo = None # Add the transaction type to the description, unless it's not useful. trntype = find_child(stmttrn, 'trntype', saxutils.unescape) if trntype in ('DEBIT', 'CREDIT'): trntype = None narration = ' / '.join(filter(None, [name, memo, trntype])) # Create a single posting for it; the user will have to manually categorize # the other side. number = find_child(stmttrn, 'trnamt', D) units = amount.Amount(number, currency) posting = data.Posting(account, units, None, None, None, None) # Build the transaction with a single leg. fileloc = data.new_metadata('', 0) return data.Transaction(fileloc, date, flag, payee, narration, data.EMPTY_SET, data.EMPTY_SET, [posting]) beangulp-0.2.0/examples/importers/runtests.sh000077500000000000000000000002121474340320100213670ustar00rootroot00000000000000#!/bin/bash # Run all the regression tests. DATA=../importers_tests python3 acme.py test $DATA/acme python3 utrade.py test $DATA/utrade beangulp-0.2.0/examples/importers/tests/000077500000000000000000000000001474340320100203105ustar00rootroot00000000000000beangulp-0.2.0/examples/importers/tests/__init__.py000066400000000000000000000000001474340320100224070ustar00rootroot00000000000000beangulp-0.2.0/examples/importers/tests/test_ofx.py000066400000000000000000000430611474340320100225210ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import datetime import re import bs4 from beancount.core.number import D from beancount.parser import parser from beancount.parser import cmptest from importers import ofx def clean_xml(string): """Compress some formatted XML as it might appear in a real file.""" return re.sub(r"(^[ \t\n]+|[ \t\n]+$)", "", string, flags=re.MULTILINE).replace('\n', '') class TestOFXImporter(cmptest.TestCase): def test_parse_ofx_time(self): dtime = datetime.datetime(2014, 1, 12, 5, 0, 0) self.assertEqual(dtime, ofx.parse_ofx_time('20140112050000.000[-7:MST]')) self.assertEqual(dtime, ofx.parse_ofx_time('20140112050000')) self.assertEqual(dtime.replace(hour=0), ofx.parse_ofx_time('20140112')) def test_find_acctids(self): contents = clean_xml(""" 0 USD 379700001111222 false """) self.assertEqual(['379700001111222'], list(ofx.find_acctids(contents))) def test_find_max_date(self): contents = clean_xml(""" 0 0 INFO USD -2356.38 20140112050000.000[-7:MST] """) date = ofx.find_max_date(contents) self.assertEqual(datetime.date(2014, 1, 12), date) def test_find_currency(self): contents = clean_xml(""" 0 USD CAD """) soup = bs4.BeautifulSoup(contents, 'lxml') self.assertEqual("USD", ofx.find_currency(soup)) def test_find_statement_transactions(self): contents = clean_xml(""" 0 0 INFO USD 379700001111222 20140112050000.000[-7:MST] 20140112050000.000[-7:MST] DEBIT 20131121000000.000[-7:MST] -29 320133250213757584 320133250213757584 JEFFREY'S 0252 NEW YORK 0000000681 646-429-8383 DEBIT 20131122000000.000[-7:MST] -13.93 320133260227320537 320133260227320537 WHOLEFDS HOU 10236 02124201320 042102720272124201320 -2356.38 20140112050000.000[-7:MST] 0 0 INFO CAD 456700001111222 20131112000000.000[-7:MST] 20131231000000.000[-7:MST] DEBIT 20131112000000.000[-7:MST] -21.51 320133160086762615 320133160086762615 LOBSTER JOINT 542929NEW YORK 000224167 6468961200 -2356.38 20140112050000.000[-7:MST] """) soup = bs4.BeautifulSoup(contents, 'lxml') txns = list(ofx.find_statement_transactions(soup)) self.assertEqual(2, len(txns)) self.assertEqual('379700001111222', txns[0][0]) self.assertIsInstance(txns[0][2][0], bs4.element.Tag) self.assertEqual(4, len(txns[0])) self.assertEqual(2, len(txns[0][2])) self.assertIsInstance(txns[1][2][0], bs4.element.Tag) self.assertEqual('456700001111222', txns[1][0]) self.assertEqual(4, len(txns[1])) self.assertEqual(1, len(txns[1][2])) def test_find_child(self): contents = clean_xml(""" DEBIT 20131122000000.000[-7:MST] -13.93 320133260227320537 320133260227320537 WHOLE & FDS HOU 10236 02124201320 042102720272124201320 """) node = bs4.BeautifulSoup(contents, 'lxml') self.assertEqual('20131122000000.000[-7:MST]', ofx.find_child(node, 'dtposted')) self.assertEqual('-13.93', ofx.find_child(node, 'trnamt')) self.assertEqual('320133260227320537', ofx.find_child(node, 'fitid')) self.assertEqual('320133260227320537', ofx.find_child(node, 'refnum')) self.assertEqual('WHOLE & FDS HOU 10236 02124201320', ofx.find_child(node, 'name')) self.assertEqual('042102720272124201320', ofx.find_child(node, 'memo')) # Test conversions. self.assertEqual(datetime.datetime(2013, 11, 22, 0, 0, 0), ofx.find_child(node, 'dtposted', ofx.parse_ofx_time)) self.assertEqual(D('-13.93'), ofx.find_child(node, 'trnamt', D)) def test_build_transaction(self): contents = clean_xml(""" DEBIT 20131122000000.000[-7:MST] -13.93 320133260227320537 320133260227320537 WHOLEFDS HOU 10236 02124201320 042102720272124201320 """) node = bs4.BeautifulSoup(contents, 'lxml') entry = ofx.build_transaction(node, '&', 'Liabilities:CreditCard', 'EUR') self.assertEqualEntries(""" 2013-11-22 & "WHOLEFDS HOU 10236 02124201320 / 042102720272124201320" Liabilities:CreditCard -13.93 EUR """, [entry]) def _extract_with_balance(self): ofx_contents = clean_xml(""" 0 INFO LOGIN SUCCESSFUL 20140112083600.212[-7:MST] ENG AMEX 3101 FMPWEB 3101 20140112083600 EXAMPLEUSER 0 0 INFO USD 379700001111222 FALSE DOWNLOAD90DAYS 379700001111222 TRUE B E308F58246398A74C52504A8B06D5F05 20140112050000.000[-7:MST] 20140112050000.000[-7:MST] DEBIT 20131124000000.000[-7:MST] -143.94 320133280255184014 320133280255184014 PRUNE NEW YORK 7101466 RESTAURANT DEBIT 20131125000000.000[-7:MST] -28.05 320133290268683266 320133290268683266 TAKAHACHI RESTAURANTNEW YORK 000451990 RESTAURANT DEBIT 20131126000000.000[-7:MST] -18.76 320133300285014247 320133300285014247 UNION MARKET - HOUSNEW YORK 47155 GROCERY STORE -2356.38 20140112050000.000[-7:MST] FALSE FALSE FALSE """) soup = bs4.BeautifulSoup(ofx_contents, 'lxml') entries, _, __ = parser.parse_string(""" 2013-11-24 * "PRUNE NEW YORK / 7101466 RESTAURANT" Liabilities:CreditCard -143.94 USD 2013-11-25 * "TAKAHACHI RESTAURANTNEW YORK / 000451990 RESTAURANT" Liabilities:CreditCard -28.05 USD 2013-11-26 * "UNION MARKET - HOUSNEW YORK / 47155 GROCERY STORE" Liabilities:CreditCard -18.76 USD """) return soup, entries def test_extract_with_balance_declared(self): soup, exp_entries = self._extract_with_balance() entries = ofx.extract(soup, 'test.ofx', '379700001111222', 'Liabilities:CreditCard', '*', ofx.BalanceType.DECLARED) balance_entries, _, __ = parser.parse_string(""" 2014-01-13 balance Liabilities:CreditCard -2356.38 USD """) self.assertEqualEntries(exp_entries + balance_entries, entries) def test_extract_with_balance_last(self): soup, exp_entries = self._extract_with_balance() entries = ofx.extract(soup, 'test.ofx', '379700001111222', 'Liabilities:CreditCard', '*', ofx.BalanceType.LAST) balance_entries, _, __ = parser.parse_string(""" 2013-11-27 balance Liabilities:CreditCard -2356.38 USD """) self.assertEqualEntries(exp_entries + balance_entries, entries) def test_two_distinct_balances(self): ofx_contents = clean_xml(""" 0 USD 379700001111222 100.00 20140101000000.000[-7:MST] 0 USD 379700001111222 200.00 20140102000000.000[-7:MST] """) soup = bs4.BeautifulSoup(ofx_contents, 'lxml') entries = ofx.extract(soup, 'test.ofx', '379700001111222', 'Liabilities:CreditCard', '*', ofx.BalanceType.DECLARED) balance_entries, _, __ = parser.parse_string(""" 2014-01-02 balance Liabilities:CreditCard 100.00 USD 2014-01-03 balance Liabilities:CreditCard 200.00 USD """, dedent=True) self.assertEqualEntries(balance_entries, entries) beangulp-0.2.0/examples/importers/utrade.py000066400000000000000000000217211474340320100210070ustar00rootroot00000000000000"""Example importer for example broker UTrade. """ __copyright__ = "Copyright (C) 2016 Martin Blais" __license__ = "GNU GPLv2" import csv import datetime import re import logging from os import path from dateutil.parser import parse from beancount.core import account from beancount.core import amount from beancount.core import data from beancount.core import flags from beancount.core import position from beancount.core.number import D from beancount.core.number import ZERO import beangulp from beangulp.testing import main class Importer(beangulp.Importer): """An importer for UTrade CSV files (an example investment bank).""" def __init__(self, currency, account_root, account_cash, account_dividends, account_gains, account_fees, account_external): self.currency = currency self.account_root = account_root self.account_cash = account_cash self.account_dividends = account_dividends self.account_gains = account_gains self.account_fees = account_fees self.account_external = account_external def identify(self, filepath): # Match if the filename is as downloaded and the header has the unique # fields combination we're looking for. if not re.match(r"UTrade\d\d\d\d\d\d\d\d\.csv", path.basename(filepath)): return False with open(filepath, 'r') as fd: head = fd.read(13) if head != "DATE,TYPE,REF": return False return True def filename(self, filepath): return 'utrade.{}'.format(path.basename(filepath)) def account(self, filepath): return self.account_root def date(self, filepath): # Extract the statement date from the filename. return datetime.datetime.strptime(path.basename(filepath), 'UTrade%Y%m%d.csv').date() def extract(self, filepath, existing): # Open the CSV file and create directives. entries = [] index = 0 with open(filepath) as infile: for index, row in enumerate(csv.DictReader(infile)): meta = data.new_metadata(filepath, index) date = parse(row['DATE']).date() rtype = row['TYPE'] link = f"ut{row['REF #']}" desc = f"({row['TYPE']}) {row['DESCRIPTION']}" units = amount.Amount(D(row['AMOUNT']), self.currency) fees = amount.Amount(D(row['FEES']), self.currency) other = amount.add(units, fees) if rtype == 'XFER': assert fees.number == ZERO txn = data.Transaction( meta, date, flags.FLAG_OKAY, None, desc, data.EMPTY_SET, {link}, [ data.Posting(self.account_cash, units, None, None, None, None), data.Posting(self.account_external, -other, None, None, None, None), ]) elif rtype == 'DIV': assert fees.number == ZERO # Extract the instrument name from its description. match = re.search(r'~([A-Z]+)$', row['DESCRIPTION']) if not match: logging.error("Missing instrument name in '%s'", row['DESCRIPTION']) continue instrument = match.group(1) account_dividends = self.account_dividends.format(instrument) txn = data.Transaction( meta, date, flags.FLAG_OKAY, None, desc, data.EMPTY_SET, {link}, [ data.Posting(self.account_cash, units, None, None, None, None), data.Posting(account_dividends, -other, None, None, None, None), ]) elif rtype in ('BUY', 'SELL'): # Extract the instrument name, number of units, and price from # the description. That's just what we're provided with (this is # actually realistic of some data from some institutions, you # have to figure out a way in your parser). match = re.search(r'\+([A-Z]+)\b +([0-9.]+)\b +@([0-9.]+)', row['DESCRIPTION']) if not match: logging.error("Missing purchase infos in '%s'", row['DESCRIPTION']) continue instrument = match.group(1) account_inst = account.join(self.account_root, instrument) units_inst = amount.Amount(D(match.group(2)), instrument) rate = D(match.group(3)) if rtype == 'BUY': cost = position.Cost(rate, self.currency, None, None) txn = data.Transaction( meta, date, flags.FLAG_OKAY, None, desc, data.EMPTY_SET, {link}, [ data.Posting(self.account_cash, units, None, None, None, None), data.Posting(self.account_fees, fees, None, None, None, None), data.Posting(account_inst, units_inst, cost, None, None, None), ]) elif rtype == 'SELL': # Extract the lot. In practice this information not be there # and you will have to identify the lots manually by editing # the resulting output. You can leave the cost.number slot # set to None if you like. match = re.search(r'\(LOT ([0-9.]+)\)', row['DESCRIPTION']) if not match: logging.error("Missing cost basis in '%s'", row['DESCRIPTION']) continue cost_number = D(match.group(1)) cost = position.Cost(cost_number, self.currency, None, None) price = amount.Amount(rate, self.currency) account_gains = self.account_gains.format(instrument) txn = data.Transaction( meta, date, flags.FLAG_OKAY, None, desc, data.EMPTY_SET, {link}, [ data.Posting(self.account_cash, units, None, None, None, None), data.Posting(self.account_fees, fees, None, None, None, None), data.Posting(account_inst, units_inst, cost, price, None, None), data.Posting(account_gains, None, None, None, None, None), ]) else: logging.error("Unknown row type: %s; skipping", rtype) continue entries.append(txn) # Insert a final balance check. if index: entries.append( data.Balance(meta, date + datetime.timedelta(days=1), self.account_cash, amount.Amount(D(row['BALANCE']), self.currency), None, None)) return entries @staticmethod def cmp(a, b): # This importer attaches an unique ID to all transactions in # the form of a link. The link can be used to implement # transaction duplicates detection based on transactions IDs. if not isinstance(a, data.Transaction): return False if not isinstance(b, data.Transaction): return False # Get all the links with the expected ut$ID format. aids = [link for link in a.links if re.match(r'ut\d{8}', link)] if not aids: # If there are no matching links, stop here. return False # Get all the links with the expected ut$ID format. bids = [link for link in b.links if re.match(r'ut\d{8}', link)] if not bids: # If there are no matching links, stop here. return False if len(aids) != len(bids): return False # Compare all collected IDs. if all(aid == bid for aid, bid in zip(sorted(aids), sorted(bids))): return True return False if __name__ == '__main__': importer = Importer( "USD", "Assets:US:UTrade", "Assets:US:UTrade:Cash", "Income:US:UTrade:{}:Dividend", "Income:US:UTrade:{}:Gains", "Expenses:Financial:Fees", "Assets:US:BofA:Checking") main(importer) beangulp-0.2.0/examples/ledger/000077500000000000000000000000001474340320100163645ustar00rootroot00000000000000beangulp-0.2.0/examples/ledger/ledger.beancount000066400000000000000000000412271474340320100215340ustar00rootroot00000000000000;; -*- mode: org; mode: beancount -*- ;; Example Beancount input file for the purpose of showing how to integrate ;; import scripts. ** General Expenses 2000-04-01 open Equity:Opening-Balances 2000-01-01 open Expenses:Books 2000-01-01 open Expenses:Communications:Phone 2000-01-01 open Expenses:Food:Grocery 2000-01-01 open Expenses:Food:Pharmacy 2000-01-01 open Expenses:Food:Restaurant 2000-01-01 open Expenses:Fun:Movie 2000-01-01 open Expenses:Fun:Music 2000-01-01 open Expenses:Financial:Fees ** BofA Checking Account 2013-10-01 open Assets:US:BofA:Checking ** BofA Credit Card Account 2013-10-01 open Liabilities:US:CreditCard 2013-10-01 pad Liabilities:US:CreditCard Equity:Opening-Balances 2013-10-26 * "DUANE READE #14354 0NEW YORK / 99999993299 8002892273" Liabilities:US:CreditCard -18.92 USD Expenses:Food:Pharmacy 2013-10-30 * "WHOLEFDS HOU 10236 02124201320 / 042902720262124201320" Liabilities:US:CreditCard -16.59 USD Expenses:Food:Grocery 2013-11-02 * "BARNES & NOBLE 2675 NEW YORK / 00001102 BOOK STORE" Liabilities:US:CreditCard -30.49 USD Expenses:Books 2013-11-04 * "UNION MARKET - HOUSNEW YORK / 44692 GROCERY STORE" Liabilities:US:CreditCard -10.77 USD Expenses:Food:Grocery 2013-11-05 * "CULL AND PISTOL NEW YORK / 85133313309 212-255-5672" Liabilities:US:CreditCard -18.45 USD Expenses:Food:Restaurant 2013-11-06 * "CAFETASIA - #2 88430NEW YORK / 4195 RESTAURANT" Liabilities:US:CreditCard -16.75 USD Expenses:Food:Restaurant 2013-11-08 * "EATALY NY NEW YORK / 84988943312 212-229-2560" Liabilities:US:CreditCard -102.1 USD Expenses:Food:Restaurant 2013-11-09 * "WHOLEFDS HOU 10236 02124201320 / 042802720252124201320" Liabilities:US:CreditCard -10.72 USD Expenses:Food:Grocery 2013-11-09 * "UNION MARKET - HOUSNEW YORK / 108039 GROCERY STORE" Liabilities:US:CreditCard -8.98 USD Expenses:Food:Grocery 2013-11-10 * "DUANE READE #14354 0NEW YORK / 99999993314 8002892273" Liabilities:US:CreditCard -14.68 USD Expenses:Food:Pharmacy 2013-11-10 * "WHOLEFDS HOU 10236 02124201320 / 042902720282124201320" Liabilities:US:CreditCard -15.9 USD Expenses:Food:Grocery 2013-11-11 * "AMAZON.COM AMZN.COM/BI / YFTFUBNN1O6 MERCHANDISE" Liabilities:US:CreditCard -172.02 USD Expenses:Books 2013-11-11 * "CAFE MOGADOR 0048 NEW YORK / 960990 212-677-2226" Liabilities:US:CreditCard -30.09 USD Expenses:Food:Restaurant 2013-11-11 * "UNION MARKET - HOUSNEW YORK / 108634 GROCERY STORE" Liabilities:US:CreditCard -15.46 USD Expenses:Food:Grocery 2013-11-12 * "LOBSTER JOINT 542929NEW YORK / 000224167 6468961200" Liabilities:US:CreditCard -21.51 USD Expenses:Food:Restaurant 2013-11-14 * "UNION MARKET - HOUSNEW YORK / 12189 GROCERY STORE" Liabilities:US:CreditCard -22.55 USD Expenses:Food:Grocery 2013-11-21 * "JEFFREY'S 0252 NEW YORK / 0000000681 646-429-8383" Liabilities:US:CreditCard -29 USD Expenses:Food:Restaurant 2013-11-22 * "WHOLEFDS HOU 10236 02124201320 / 042102720272124201320" Liabilities:US:CreditCard -13.93 USD Expenses:Food:Grocery 2013-11-23 * "AMAZON.COM AMZN.COM/BI / V7P27IJ69JX MERCHANDISE" Liabilities:US:CreditCard -54.99 USD Expenses:Books 2013-11-24 * "PRUNE NEW YORK / 7101466 RESTAURANT" Liabilities:US:CreditCard -143.94 USD Expenses:Food:Restaurant 2013-11-25 * "TAKAHACHI RESTAURANTNEW YORK / 000451990 RESTAURANT" Liabilities:US:CreditCard -28.05 USD Expenses:Food:Restaurant 2013-11-26 * "UNION MARKET - HOUSNEW YORK / 47155 GROCERY STORE" Liabilities:US:CreditCard -18.76 USD Expenses:Food:Grocery 2013-11-29 * "WHOLEFDS HOU 10236 02124201320 / 042802720282124201320" Liabilities:US:CreditCard -23.18 USD Expenses:Food:Grocery 2013-11-29 * "T-MOBILE RECURNG PMTT-MOBILE / 1070888371 828422957 98006" Liabilities:US:CreditCard -61.98 USD Expenses:Communications:Phone 2013-12-01 * "SPOTIFY USA 28770130901 / 2600720879 WWW.SPOTIFY.COM" Liabilities:US:CreditCard -9.99 USD Expenses:Fun:Music 2013-12-02 * "GOAT TOWN 1200000549NEW YORK / 071000163 2126873641" Liabilities:US:CreditCard -61.71 USD Expenses:Food:Restaurant 2013-12-02 * "AMC VILLAGE 7 #2110 NEW YORK / 12010365699 212-982-2116" Liabilities:US:CreditCard -17.75 USD Expenses:Fun:Movie 2013-12-04 balance Liabilities:US:CreditCard -2093.01 USD ** UTrade Investment Account 2014-04-01 open Assets:US:UTrade:Cash 2015-02-01 open Assets:US:UTrade:BAPL 2015-02-01 open Income:US:UTrade:BAPL:Dividend 2015-02-01 open Income:US:UTrade:BAPL:Gains 2014-04-01 open Assets:US:UTrade:CSKO 2014-04-01 open Income:US:UTrade:CSKO:Dividend 2014-04-01 open Income:US:UTrade:CSKO:Gains 2014-05-01 open Assets:US:UTrade:HOOL 2014-05-01 open Income:US:UTrade:HOOL:Dividend 2014-05-01 open Income:US:UTrade:HOOL:Gains 2014-05-01 open Assets:US:UTrade:MSFX 2014-05-01 open Income:US:UTrade:MSFX:Dividend 2015-05-01 open Income:US:UTrade:MSFX:Gains 2014-04-01 pad Assets:US:UTrade:Cash Equity:Opening-Balances 2014-04-15 balance Assets:US:UTrade:Cash 25674.63 USD 2014-04-14 * "(BUY) BOUGHT +CSKO 50 @98.35" ^ut14167001 Assets:US:UTrade:Cash -4925.45 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 50 CSKO {98.35 USD} 2014-05-08 * "(BUY) BOUGHT +HOOL 121 @79.11" ^ut12040838 Assets:US:UTrade:Cash -9580.26 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 121 HOOL {79.11 USD} 2014-05-11 * "(BUY) BOUGHT +MSFX 104 @64.39" ^ut41579908 Assets:US:UTrade:Cash -6704.51 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 104 MSFX {64.39 USD} 2014-05-22 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut54857517 Assets:US:UTrade:Cash 28.56 USD Income:US:UTrade:HOOL:Dividend -28.56 USD 2014-05-23 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut27634682 Assets:US:UTrade:Cash 7148.74 USD Assets:US:BofA:Checking -7148.74 USD 2014-05-25 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut31749124 Assets:US:UTrade:Cash 9.63 USD Income:US:UTrade:CSKO:Dividend -9.63 USD 2014-05-28 * "(BUY) BOUGHT +HOOL 92 @52.10" ^ut83788120 Assets:US:UTrade:Cash -4801.15 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 92 HOOL {52.10 USD} 2014-05-29 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut97871874 Assets:US:UTrade:Cash 7.49 USD Income:US:UTrade:HOOL:Dividend -7.49 USD 2014-06-05 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut85665025 Assets:US:UTrade:Cash 16.32 USD Income:US:UTrade:CSKO:Dividend -16.32 USD 2014-06-14 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut31730597 Assets:US:UTrade:Cash 15.60 USD Income:US:UTrade:HOOL:Dividend -15.60 USD 2014-06-21 * "(BUY) BOUGHT +MSFX 101 @78.00" ^ut22346704 Assets:US:UTrade:Cash -7885.95 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 101 MSFX {78.00 USD} 2014-06-23 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut36811051 Assets:US:UTrade:Cash 8.70 USD Income:US:UTrade:HOOL:Dividend -8.70 USD 2014-07-01 * "(DIV) ORDINARY DIVIDEND~MSFX" ^ut30631356 Assets:US:UTrade:Cash 9.39 USD Income:US:UTrade:MSFX:Dividend -9.39 USD 2014-07-12 * "(DIV) ORDINARY DIVIDEND~MSFX" ^ut33403638 Assets:US:UTrade:Cash 16.64 USD Income:US:UTrade:MSFX:Dividend -16.64 USD 2014-07-13 balance Assets:US:UTrade:Cash 3963.83 USD 2014-07-28 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut23571586 Assets:US:UTrade:Cash 15.60 USD Income:US:UTrade:CSKO:Dividend -15.60 USD 2014-08-07 * "(BUY) BOUGHT +CSKO 48 @52.93" ^ut90404110 Assets:US:UTrade:Cash -2548.59 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 48 CSKO {52.93 USD} 2014-08-22 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut70106713 Assets:US:UTrade:Cash 22.58 USD Income:US:UTrade:HOOL:Dividend -22.58 USD 2014-08-26 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut18874360 Assets:US:UTrade:Cash 8.46 USD Income:US:UTrade:CSKO:Dividend -8.46 USD 2014-09-24 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut18146488 Assets:US:UTrade:Cash 7.36 USD Income:US:UTrade:CSKO:Dividend -7.36 USD 2014-09-25 balance Assets:US:UTrade:Cash 1469.24 USD 2014-10-13 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut42072631 Assets:US:UTrade:Cash 11.39 USD Income:US:UTrade:CSKO:Dividend -11.39 USD 2014-10-14 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut45612217 Assets:US:UTrade:Cash 17.45 USD Income:US:UTrade:HOOL:Dividend -17.45 USD 2014-10-22 * "(SELL) SOLD +CSKO 22 @88.13 (LOT 98.35)" ^ut84625538 Assets:US:UTrade:Cash 1930.91 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 22 CSKO {98.35 USD} @ 88.13 USD Income:US:UTrade:CSKO:Gains 2014-11-09 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut65176118 Assets:US:UTrade:Cash 7.01 USD Income:US:UTrade:HOOL:Dividend -7.01 USD 2014-11-27 * "(SELL) SOLD +CSKO 93 @33.31 (LOT 32.59)" ^ut92392307 Assets:US:UTrade:Cash 3089.88 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 93 CSKO {32.59 USD} @ 33.31 USD Income:US:UTrade:CSKO:Gains 2014-12-01 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut10447525 Assets:US:UTrade:Cash 6.82 USD Income:US:UTrade:CSKO:Dividend -6.82 USD 2014-12-02 * "(BUY) BOUGHT +HOOL 55 @66.61" ^ut74330963 Assets:US:UTrade:Cash -3671.50 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 55 HOOL {66.61 USD} 2014-12-13 * "(SELL) SOLD +HOOL 50 @39.13 (LOT 42.33)" ^ut79580321 Assets:US:UTrade:Cash 1948.55 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 50 HOOL {42.33 USD} @ 39.13 USD Income:US:UTrade:HOOL:Gains 2014-12-14 balance Assets:US:UTrade:Cash 4809.75 USD 2014-12-20 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut60292945 Assets:US:UTrade:Cash 5.17 USD Income:US:UTrade:CSKO:Dividend -5.17 USD 2014-12-22 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut98071623 Assets:US:UTrade:Cash 19.97 USD Income:US:UTrade:HOOL:Dividend -19.97 USD 2014-12-31 * "(BUY) BOUGHT +MSFX 87 @50.34" ^ut94789086 Assets:US:UTrade:Cash -4387.53 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 87 MSFX {50.34 USD} 2015-01-05 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut89525843 Assets:US:UTrade:Cash 23.67 USD Income:US:UTrade:CSKO:Dividend -23.67 USD 2015-01-17 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut92120597 Assets:US:UTrade:Cash 4876.00 USD Assets:US:BofA:Checking -4876.00 USD 2015-01-19 * "(BUY) BOUGHT +HOOL 40 @99.15" ^ut76302399 Assets:US:UTrade:Cash -3973.95 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 40 HOOL {99.15 USD} 2015-02-02 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut57564142 Assets:US:UTrade:Cash 7.98 USD Income:US:UTrade:HOOL:Dividend -7.98 USD 2015-02-06 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut90692243 Assets:US:UTrade:Cash 12.27 USD Income:US:UTrade:BAPL:Dividend -12.27 USD 2015-02-20 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut34606455 Assets:US:UTrade:Cash 14.83 USD Income:US:UTrade:BAPL:Dividend -14.83 USD 2015-02-22 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut16360724 Assets:US:UTrade:Cash 24.24 USD Income:US:UTrade:BAPL:Dividend -24.24 USD 2015-02-23 balance Assets:US:UTrade:Cash 1432.40 USD 2015-02-26 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut39293066 Assets:US:UTrade:Cash 10.71 USD Income:US:UTrade:HOOL:Dividend -10.71 USD 2015-03-06 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut95397432 Assets:US:UTrade:Cash 5393.93 USD Assets:US:BofA:Checking -5393.93 USD 2015-03-20 * "(SELL) SOLD +BAPL 107 @42.87 (LOT 38.83)" ^ut47069288 Assets:US:UTrade:Cash 4579.14 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:BAPL 107 BAPL {38.83 USD} @ 42.87 USD Income:US:UTrade:BAPL:Gains 2015-03-24 * "(BUY) BOUGHT +BAPL 89 @56.05" ^ut91955231 Assets:US:UTrade:Cash -4996.40 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:BAPL 89 BAPL {56.05 USD} 2015-03-30 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut68299938 Assets:US:UTrade:Cash 3944.15 USD Assets:US:BofA:Checking -3944.15 USD 2015-04-01 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut11669685 Assets:US:UTrade:Cash 4.69 USD Income:US:UTrade:HOOL:Dividend -4.69 USD 2015-04-02 * "(BUY) BOUGHT +CSKO 149 @63.85" ^ut35067825 Assets:US:UTrade:Cash -9521.60 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 149 CSKO {63.85 USD} 2015-04-14 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut26605450 Assets:US:UTrade:Cash 9.95 USD Income:US:UTrade:BAPL:Dividend -9.95 USD 2015-04-15 balance Assets:US:UTrade:Cash 856.97 USD 2015-05-17 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut57567122 Assets:US:UTrade:Cash 15.96 USD Income:US:UTrade:HOOL:Dividend -15.96 USD 2015-05-18 * "(SELL) SOLD +BAPL 146 @32.01 (LOT 26.15)" ^ut98562913 Assets:US:UTrade:Cash 4665.51 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:BAPL 146 BAPL {26.15 USD} @ 32.01 USD Income:US:UTrade:BAPL:Gains 2015-05-19 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut45119594 Assets:US:UTrade:Cash 13.91 USD Income:US:UTrade:HOOL:Dividend -13.91 USD 2015-05-26 * "(SELL) SOLD +CSKO 89 @66.30 (LOT 63.85)" ^ut53367649 Assets:US:UTrade:Cash 5892.75 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 89 CSKO {63.85 USD} @ 66.30 USD Income:US:UTrade:CSKO:Gains 2015-06-07 * "(BUY) BOUGHT +HOOL 777 @5.16" ^ut56695140 Assets:US:UTrade:Cash -4017.27 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 777 HOOL {5.16 USD} 2015-06-19 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut17207133 Assets:US:UTrade:Cash 12.79 USD Income:US:UTrade:BAPL:Dividend -12.79 USD 2015-06-20 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut50840576 Assets:US:UTrade:Cash 13.89 USD Income:US:UTrade:BAPL:Dividend -13.89 USD 2015-06-25 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut88647893 Assets:US:UTrade:Cash 10.13 USD Income:US:UTrade:BAPL:Dividend -10.13 USD 2015-06-29 * "(BUY) BOUGHT +CSKO 54 @64.93" ^ut22740217 Assets:US:UTrade:Cash -3514.17 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 54 CSKO {64.93 USD} 2015-07-02 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut27217440 Assets:US:UTrade:Cash 26.19 USD Income:US:UTrade:HOOL:Dividend -26.19 USD 2015-07-18 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut64873832 Assets:US:UTrade:Cash 9.80 USD Income:US:UTrade:BAPL:Dividend -9.80 USD 2015-07-19 * "(BUY) BOUGHT +HOOL 31 @97.08" ^ut51600680 Assets:US:UTrade:Cash -3017.43 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 31 HOOL {97.08 USD} 2015-07-20 balance Assets:US:UTrade:Cash 969.03 USD 2015-07-26 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut75756901 Assets:US:UTrade:Cash 14.45 USD Income:US:UTrade:HOOL:Dividend -14.45 USD 2015-08-18 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut28729581 Assets:US:UTrade:Cash 8.15 USD Income:US:UTrade:CSKO:Dividend -8.15 USD 2015-08-31 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut97130478 Assets:US:UTrade:Cash 11.21 USD Income:US:UTrade:BAPL:Dividend -11.21 USD 2015-09-13 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut74624169 Assets:US:UTrade:Cash 12.47 USD Income:US:UTrade:CSKO:Dividend -12.47 USD 2015-09-19 * "(DIV) ORDINARY DIVIDEND~MSFX" ^ut86506548 Assets:US:UTrade:Cash 14.07 USD Income:US:UTrade:MSFX:Dividend -14.07 USD 2015-09-20 balance Assets:US:UTrade:Cash 1029.38 USD 2015-10-21 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut71819731 Assets:US:UTrade:Cash 7.98 USD Income:US:UTrade:HOOL:Dividend -7.98 USD 2015-10-26 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut99810091 Assets:US:UTrade:Cash 16.83 USD Income:US:UTrade:HOOL:Dividend -16.83 USD 2015-11-15 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut84191955 Assets:US:UTrade:Cash 22.75 USD Income:US:UTrade:BAPL:Dividend -22.75 USD 2015-11-17 * "(SELL) SOLD +MSFX 41 @84.22 (LOT 93.91)" ^ut35166597 Assets:US:UTrade:Cash 3445.07 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 41 MSFX {93.91 USD} @ 84.22 USD Income:US:UTrade:MSFX:Gains 2015-12-03 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut48233019 Assets:US:UTrade:Cash 9.01 USD Income:US:UTrade:CSKO:Dividend -9.01 USD 2015-12-04 balance Assets:US:UTrade:Cash 4531.02 USD ;; NEW TRANSACTIONS NEED BE INSERTED HERE. beangulp-0.2.0/examples/tests/000077500000000000000000000000001474340320100162645ustar00rootroot00000000000000beangulp-0.2.0/examples/tests/acme/000077500000000000000000000000001474340320100171715ustar00rootroot00000000000000beangulp-0.2.0/examples/tests/acme/acmebank1.pdf000066400000000000000000000414141474340320100215120ustar00rootroot00000000000000%PDF-1.4 %äüöß 2 0 obj <> stream xSKk0 W\Hƒ-ہ`$M²:{ؿnA קOm/آ9̯՚Su* M2=;Lf7a}g!YNNG6ؘɶaR Su6lIJcy5O7cg3BĥHB"d%:hBCsg-eٻrkpЬ%@/ HaWGC_vp9 4D"YD'Uz]DRH8Pl[';Jo#^c1ZL8$3uʕp>̞YB knW3 =WqxqC rH= L.%Xu5 G8Y<7U4!?+"*#sJ 0JEy2f7\ߒycvٛ9U/= endstream endobj 3 0 obj 435 endobj 4 0 obj <> /Length 8 /Filter/FlateDecode >> stream x endstream endobj 5 0 obj <> endobj 7 0 obj <> stream xݼ tǵ(ZU=k4MݣHQ#$,%@|c} 6&sbq8p,/19q|؎'<kv08ZﾵZoڵk׮]v]]؎dDAm}#R!m qQ 1y°Fl6rgy9\'¢PqIi=(# Q7}SCo~^8zE_LA !07~~~X.-to=Ay.G]}AvS+q }^UFbݍ^n"fx 7ZJ.CQZBD@8O8> kf`k_0RtZP{IF׎P]^m@Ð > c'2PggRuvY*remKZ[7.R TWU/ ei^ ɠZ 2f%!&I6OJh^D DRT4I17S*@99J%M\Ă-,)%1j$]:~IZ'&rfԌ 267I-;7lN4bi񀾤4,FN(VRм4A6d}&o xI,5Eh2YԪ,!utH<]Rs"ss0KPz;JJ_fOF_hIdXI:Ic@SRLX4 5x*\K8lTeySIŤcQ*\ߔAX & f%QV5=\b /9HKz$ +5VzZ`ÊVӊBflJKJk:`AA۲6XR Ƭ[uZ_^3cѾEwwk_;nיmԛ\}dt8mSè!o7BTDS ֘aL&>*)eguݱxx+Ffqq1RaTȍ0&I2)DLbbtfalnE؂Nsn\qfJy'$0`Ñ2P [+:pj3YGJ9ϦOY̾AϿ..AM*k8G8^Ͱfil_R4"%iLpeWRkaD 7lX%,>L=M+ڰ+G `Է1YA#hA)O&ͯ(1Gw6Xj~d¯X g>:jm2ceP_aFq_Q#q=yLwxz2wXZ0f)[+\)Kh}b #ZG(&E"i=G1R_VD}R=WOU'菅?'z\EJOs x=A@}Qj5 *Eɓz5 {sAe@ RGz{GM.*V( ]tSW*]S4rbi]t64/Ob~ Rkov(=6t<9s*ӬS++\27T}GΗ[g. .v5\|{ZcI敕&]Ś[VCmvl}f!*BUJ`KCnbvE]&,1F6lbbNOga9VMd J `~#Jݯ7W srPUduyq$¤$ RF! HuLx VSofy"6*qi,Xb WU)%;R)%]][AslCp^͢Co[&prY 6X M .5d;wjvrYb/G,Xw2e^Bz6`SR3gG)0 n{7La>Է0m"c}Rލ<y7xWz q8ax5ui uri=ݑ^=Fn^^UqJVZ)+w5QW\̯ktm"3{55>N񃥞PJw0/Cvy.˓UhdL1UKc4UE .5b#hVY]EKN h/ WPd5}O[*jD|S1SYgXwncc4۽hEgɎB۽jS_YUYC1+ے9^x]& zs5~"e\Nao5yML!iqK  d녘b^IJnIYl-Xz!SV % BLB́'GLRTH^].oI=Fz=Kqή .10q 5DuV'B%c\?l=ŢlO:)8`=,!?T-H.|BzWy~180X_]AR,,%0"|m`0cFK?ƮL/.ѥzRV@t9AgSqƝ,,b`1vTC FVa\ez,J! =QJX3R#_uЇ=N5iiB3#C${Ӛ[,7~yߗ7/֍Ϸ-'K̫߸gߞ#zm]63Z$aMP: wƇZZ { G 1`^n^\cT>YNj)T~Krr9( 9\oi*_C.OQ5tV%~X3Fi2?e6뤵ROgmҼ6D*6Pd4>Rvyzʶ F zJm>1zwv`09TV8Ŭ<*XyݛTOCy@> }l%\Q%*J+> \"X{ɅIțb^rR@1lQy< ԥ`0=57~M*~uE.^shX1scXc4pn%^1V3ZXXժ=%QbMr9=F/FSQ|4(̓Q|1zry>OFqUtmtrr⏢b%{xghSEM%^2O (x7F&EQ(T=[%M @e'{*|rm_i>2'̱(KIvUQB(!=^%9%t;\^]8vؔіQ9lB/ 3ѩ(2t/0'K(>v.%J4kRrMDOD ̑B? (NɛP۬by*GGdD1"#5B=K.p5|W.&rC=Yu`YBM%+tT%C7Lו3`5hAV/\F7{sN \/ \;hY]nz:ݩ:S::r*mڅ鰢֖/:n .z[購lxeXC"$-Ybf N!t/ 1ӊnbRQd18#bflC;EHuxcznOHuԹV0HvP7F=iSKEZS@ER澺r[/1WLZF6&CkϸyliVI*i_Y%] 6E a:7 m91B,1=ԺZY0)%YÛٽDkvbg9h ,qEgjC7rP o7xhAԹjny)73sS>83 B<6+&Da=M9u rX{H-fth?GEƩ! wF;S."Pp\+s#$Gu\2ghVQ7ܵZmNWc3[-sBD3fbU]Rc$ݧG7KĶ`qkn]н8&5J-ye/Xe߃AWfiŦïi&Yԧ:]`>LöZ ZYL}/,4-A' 2z|byIHR$H $V# 1oJJui@z!铛P;4`jۘ" ʭO_[!OlE)@eD!܁6lln֐N.nTUE5FkNg(W6Ȫe, 0?}evϭ V-{x[oGOyG/=6w0Ƴ0}\ a= Ƹč#ƤEÈS8wKri'č3gJe90A+Wql c{zBWunwv}[cL𼼂~pT)%JiDt)[0""y9LZł- 뛮mpCo\In( g_,Y,fȩڝgv|s]Ƒ6u䙙m%͌PǶ/t2K7%+&V ٶ^-v_D}>]>䵵 / X-_5?u~j8oe}}s+yir%Nu렋"H0,^YApr͂?X0hoAb5K,7[Y[\5SҒc0Kx")!U ' h纾V &KCjR_E"5ȟs*#껑|g:0<}_Ox EM{":;ol.|P["p7`UGb_n®Jd7͝o*+ [ս Y>ڦ7g"c>=R8YxRBN=Mz +$k}>߯'"[E%A}c#|";z]bahu(8ThcZ׹k>~w؍<?+5Ψqb=ʎ| e Zj?Vϼ'{!isw<`iE{Pjc?wIG~|ďIŸO#?=D+)iWM$ĸ56;8{{`n ȏPNDC,ѱx/9J`A "t9%\ƞ VdDMZd\15qƕq#1usS0[Tå+Q_ͮ_4Qr$.o~0ROPuУ}R[1sijV 1XnrX1-/3 zTKK<3B"ap2TaWbg%|c>;J-#!tX [\5q$v1- ^Рp? }W:-YJ`0(V뽼dֶ/ݫ(toO>rho{tkoLeϾfUr.t"O:0=>:ɤπG@WctzN'x5[`B`¹*dtހ֙yH`ߍ%b RXĦ|V(.0'EJ߼ B> dR$!gg(:tϰ+bB |zʁ z"Z~]ޝCn> Ir4&&өKSw% `μeM8G\qkՌնu^8+'S'$\l\"?zJ>}SLcmj#Ԍez\(\ka ֣-N8bh=zX{"7 JĀ 2!i2hp2[%4uqq&cq`Bc[Rqni"ٓja>w&Nl&\YyYg^RJ lVrrsZu|.:Ǘv9nv>Lfu֘ՁlH!R|CniG.ke OޓQ2sHke #YzyL2!!AXf)e1eO\2?&+2y6]rڰNd!Nnjȝ!hqL^YZ,q,M防O^eBd_/G28D\n\jI?a1(v&ϓeG?|Q&O{eZ LD',Ǜc|V&s,)%Qh1]1!V+O̘]B{T,T+; Xe]$c2"OG<%syLxXfTјܢ(;f[pCt]|D_Y⣜)c7:|}>|rrw\XUFM2;,{үF;#h dCL=7Sn8io zޫ5.+}/mi#|,qyဿ:RWN=g>~U^Mup$y=1>&V`6@k@?PQIe0&n4i:bbL~#DaMxtDΚ.^71&FIMGiowxo`QV_{JCGauoлx c/@\u,_gE.DFEBTt[{jQX E8^G&1EKQ*A^Nɂ,’$O^ )qUѫ 8=9Ťx47HxFCZ쐄+&_bJK$vD@ Z1H^䪄/Z3')ȩMgbj'Ԭ% }$'$,Kt*; :(IqiR:"%i )2rZ͙Izs&';QvXlq]:|"Rz^U;Ee;t)mh,mF?#Q1tK՟;4TSOVXRҒ3Dˑ+)9?o1ީ7<ǁ@ojL-#qTWFX=Ʌ\N}딨ͷF1V3ƿ5γ6ZWP̭p}E>&[^<ڳ3Ě-lrdr]pR%B6.Z*{@SyⱛHJJ% ⛛fEC}mOֽL{<~tg0\#3[5._RӬ0k5XC_1A $.Iwyc&Vqxb^f "Bd I;`VCf6DGYP$Dbd TbX̷R_R"~~1?;̔t^ν=xc?:._8w{9P?YNrs~tUhxHkj+[0|Q32E% hKupQ@lP_^U d'fԑZ\ҧqt?Zbi-T+ YAƇڡ _#ԡ4^-~xv m|'| 2P~0[Ιܯ5iIBY!nH2@-X>>f=k`YYY?{5zH bwZ kGŕ^m⺜1P&20AZ4؎ve`hX2YP2kBA.Xxy6@_'x f%h:M#B4L) b  ̡X֠\ E120st1P~6Ft+7߄~=K;ڴyB,X(UXDdRq֭J0. /-il^}и'Nl"\І⚁vl`{X"~c434YNG'9](]S*&O}WmP&xxb3tcCCikGxxld8ӥ򾉉'&Fûv*oҍ]Ma䥛'm]>ߡQj-ariе6Nm 2Nھ)%^m ܍ڄ6 $.BZ+ |8Fb@U /,[!øtҝj]J j5¾J  1phmߠCK~;Q1CP܄v@)EFlW[_=WVƯ̇~Q{$&˄;M95*U\Ie1]jWBPJ3ʍ* ȧ99#@cjzsczNJw;6 Uhf57}͜7f6ZoVȈ*u7mzKU@e$]{*71-k-j^9[oCg\3TIKm Uڥ*v:!!nM{3חei/%~D}^Of8\kx#Zꅫd<S|y.wJȕW4z؈O߉[m0ɷ=vZL[/LSeS#SSN?8GgVsI<-!#O`'O0=Z5ccW+-=~d?3> ;x?=ZD:Y"Mӆ`AKW2WkupY-i`V2aWs;'oґK%-Xko NV$> endobj 10 0 obj <> stream x]n0E .E q(Q `pu4vĔ@ }9s8syLa{ÜcwٜGر9e%ݼS6j?m8y9>M|0"k9|mosz>i{ϩ_cbCPƞoSqlÅUQ4f7F}1TZm ;VW•2IV ߄ר/ߔk j4;J}smzq`_+I# wo¿O Z3WaSA^_s-gkɤŇ5'1+RςW?i&I3W;_w:Tx[cL#/ΪL{4t7 endstream endobj 11 0 obj <> endobj 12 0 obj <> endobj 13 0 obj <> /ExtGState<> /ProcSet[/PDF/Text/ImageC/ImageI/ImageB] >> endobj 1 0 obj <>/Contents 2 0 R>> endobj 6 0 obj <> endobj 14 0 obj <> endobj 15 0 obj < /Producer /CreationDate(D:20160321005904-04'00')>> endobj xref 0 16 0000000000 65535 f 0000016126 00000 n 0000000019 00000 n 0000000525 00000 n 0000000545 00000 n 0000000724 00000 n 0000016269 00000 n 0000000764 00000 n 0000014901 00000 n 0000014923 00000 n 0000015118 00000 n 0000015617 00000 n 0000015968 00000 n 0000016001 00000 n 0000016368 00000 n 0000016465 00000 n trailer < ] /DocChecksum /0A205D5467C5CF93B4FAD5D9CA89AC07 >> startxref 16640 %%EOF beangulp-0.2.0/examples/tests/acme/acmebank1.pdf.beancount000066400000000000000000000001111474340320100234540ustar00rootroot00000000000000;; Account: Assets:US:ACMEBank ;; Date: 2015-09-30 ;; Name: acmebank.pdf beangulp-0.2.0/examples/tests/csvbank/000077500000000000000000000000001474340320100177135ustar00rootroot00000000000000beangulp-0.2.0/examples/tests/csvbank/statement.csv000066400000000000000000000012711474340320100224350ustar00rootroot00000000000000Details,Posting Date,"Description",Amount,Type,Balance,Check or Slip #, DEBIT,3/18/2016,"Payment to Chafe card ending in 1234 03/18",-2680.89,ACCT_XFER,3409.86,, CREDIT,3/15/2016,"EMPLOYER INC DIRECT DEP PPD ID: 1111111111",2590.73,ACH_CREDIT,6090.75,, DEBIT,3/14/2016,"INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789",-150.00,ACH_DEBIT,3500.02,, DEBIT,3/6/2016,"ATM WITHDRAWAL 001234 03/8888 DELANC",-60.00,ATM,3650.02,, CREDIT,3/5/2016,"CA STATE NYSTTAXRFD PPD ID: 1111111111",110.00,ACH_CREDIT,3710.02,, DEBIT,3/4/2016,"BOOGLE WALLET US000NEI9T WEB ID: C234567890",-1300.00,ACH_DEBIT,3600.02,, beangulp-0.2.0/examples/tests/csvbank/statement.csv.beancount000066400000000000000000000014511474340320100244120ustar00rootroot00000000000000;; Account: Assets:US:CSVBank ;; Date: 2016-03-18 ;; Name: csvbank.statement.csv 2016-03-04 * "BOOGLE WALLET US000NEI9T WEB ID: C234567890" Assets:US:CSVBank -1300.00 USD 2016-03-05 * "CA STATE NYSTTAXRFD PPD ID: 1111111111" Assets:US:CSVBank 110.00 USD 2016-03-06 * "ATM WITHDRAWAL 001234 03/8888 DELANC" Assets:US:CSVBank -60.00 USD 2016-03-14 * "INVESTMENT SEC TRANSFER A5144608 WEB ID: 1234456789" Assets:US:CSVBank -150.00 USD 2016-03-15 * "EMPLOYER INC DIRECT DEP PPD ID: 1111111111" Assets:US:CSVBank 2590.73 USD 2016-03-18 * "Payment to Chafe card ending in 1234 03/18" Assets:US:CSVBank -2680.89 USD 2016-03-19 balance Assets:US:CSVBank 3409.86 USD beangulp-0.2.0/examples/tests/utrade/000077500000000000000000000000001474340320100175505ustar00rootroot00000000000000beangulp-0.2.0/examples/tests/utrade/UTrade20140713.csv000066400000000000000000000017461474340320100223030ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-04-14,BUY,14167001,BOUGHT +CSKO 50 @98.35,7.95,-4925.45,25674.63 2014-05-08,BUY,12040838,BOUGHT +HOOL 121 @79.11,7.95,-9580.26,16094.37 2014-05-11,BUY,41579908,BOUGHT +MSFX 104 @64.39,7.95,-6704.51,9389.86 2014-05-22,DIV,54857517,ORDINARY DIVIDEND~HOOL,0,28.56,9418.42 2014-05-23,XFER,27634682,CLIENT REQUESTED ELECTRONIC FUNDING,0,7148.74,16567.16 2014-05-25,DIV,31749124,ORDINARY DIVIDEND~CSKO,0,9.63,16576.79 2014-05-28,BUY,83788120,BOUGHT +HOOL 92 @52.10,7.95,-4801.15,11775.64 2014-05-29,DIV,97871874,ORDINARY DIVIDEND~HOOL,0,7.49,11783.13 2014-06-05,DIV,85665025,ORDINARY DIVIDEND~CSKO,0,16.32,11799.45 2014-06-14,DIV,31730597,ORDINARY DIVIDEND~HOOL,0,15.60,11815.05 2014-06-21,BUY,22346704,BOUGHT +MSFX 101 @78.00,7.95,-7885.95,3929.10 2014-06-23,DIV,36811051,ORDINARY DIVIDEND~HOOL,0,8.70,3937.80 2014-07-01,DIV,30631356,ORDINARY DIVIDEND~MSFX,0,9.39,3947.19 2014-07-12,DIV,33403638,ORDINARY DIVIDEND~MSFX,0,16.64,3963.83 beangulp-0.2.0/examples/tests/utrade/UTrade20140713.csv.beancount000066400000000000000000000046041474340320100242540ustar00rootroot00000000000000;; Account: Assets:US:UTrade ;; Date: 2014-07-13 ;; Name: utrade.UTrade20140713.csv 2014-04-14 * "(BUY) BOUGHT +CSKO 50 @98.35" ^ut14167001 Assets:US:UTrade:Cash -4925.45 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 50 CSKO {98.35 USD} 2014-05-08 * "(BUY) BOUGHT +HOOL 121 @79.11" ^ut12040838 Assets:US:UTrade:Cash -9580.26 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 121 HOOL {79.11 USD} 2014-05-11 * "(BUY) BOUGHT +MSFX 104 @64.39" ^ut41579908 Assets:US:UTrade:Cash -6704.51 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 104 MSFX {64.39 USD} 2014-05-22 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut54857517 Assets:US:UTrade:Cash 28.56 USD Income:US:UTrade:HOOL:Dividend -28.56 USD 2014-05-23 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut27634682 Assets:US:UTrade:Cash 7148.74 USD Assets:US:BofA:Checking -7148.74 USD 2014-05-25 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut31749124 Assets:US:UTrade:Cash 9.63 USD Income:US:UTrade:CSKO:Dividend -9.63 USD 2014-05-28 * "(BUY) BOUGHT +HOOL 92 @52.10" ^ut83788120 Assets:US:UTrade:Cash -4801.15 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 92 HOOL {52.10 USD} 2014-05-29 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut97871874 Assets:US:UTrade:Cash 7.49 USD Income:US:UTrade:HOOL:Dividend -7.49 USD 2014-06-05 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut85665025 Assets:US:UTrade:Cash 16.32 USD Income:US:UTrade:CSKO:Dividend -16.32 USD 2014-06-14 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut31730597 Assets:US:UTrade:Cash 15.60 USD Income:US:UTrade:HOOL:Dividend -15.60 USD 2014-06-21 * "(BUY) BOUGHT +MSFX 101 @78.00" ^ut22346704 Assets:US:UTrade:Cash -7885.95 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 101 MSFX {78.00 USD} 2014-06-23 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut36811051 Assets:US:UTrade:Cash 8.70 USD Income:US:UTrade:HOOL:Dividend -8.70 USD 2014-07-01 * "(DIV) ORDINARY DIVIDEND~MSFX" ^ut30631356 Assets:US:UTrade:Cash 9.39 USD Income:US:UTrade:MSFX:Dividend -9.39 USD 2014-07-12 * "(DIV) ORDINARY DIVIDEND~MSFX" ^ut33403638 Assets:US:UTrade:Cash 16.64 USD Income:US:UTrade:MSFX:Dividend -16.64 USD 2014-07-13 balance Assets:US:UTrade:Cash 3963.83 USD beangulp-0.2.0/examples/tests/utrade/UTrade20150225.csv000066400000000000000000000017561474340320100223030ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2014-11-27,SELL,92392307,SOLD +CSKO 93 @33.31 (LOT 32.59),7.95,3089.88,6525.88 2014-12-01,DIV,10447525,ORDINARY DIVIDEND~CSKO,0,6.82,6532.70 2014-12-02,BUY,74330963,BOUGHT +HOOL 55 @66.61,7.95,-3671.50,2861.20 2014-12-13,SELL,79580321,SOLD +HOOL 50 @39.13 (LOT 42.33),7.95,1948.55,4809.75 2014-12-20,DIV,60292945,ORDINARY DIVIDEND~CSKO,0,5.17,4814.92 2014-12-22,DIV,98071623,ORDINARY DIVIDEND~HOOL,0,19.97,4834.89 2014-12-31,BUY,94789086,BOUGHT +MSFX 87 @50.34,7.95,-4387.53,447.36 2015-01-05,DIV,89525843,ORDINARY DIVIDEND~CSKO,0,23.67,471.03 2015-01-17,XFER,92120597,CLIENT REQUESTED ELECTRONIC FUNDING,0,4876.00,5347.03 2015-01-19,BUY,76302399,BOUGHT +HOOL 40 @99.15,7.95,-3973.95,1373.08 2015-02-02,DIV,57564142,ORDINARY DIVIDEND~HOOL,0,7.98,1381.06 2015-02-06,DIV,90692243,ORDINARY DIVIDEND~BAPL,0,12.27,1393.33 2015-02-20,DIV,34606455,ORDINARY DIVIDEND~BAPL,0,14.83,1408.16 2015-02-22,DIV,16360724,ORDINARY DIVIDEND~BAPL,0,24.24,1432.40 beangulp-0.2.0/examples/tests/utrade/UTrade20150225.csv.beancount000066400000000000000000000047771474340320100242660ustar00rootroot00000000000000;; Account: Assets:US:UTrade ;; Date: 2015-02-25 ;; Name: utrade.UTrade20150225.csv 2014-11-27 * "(SELL) SOLD +CSKO 93 @33.31 (LOT 32.59)" ^ut92392307 Assets:US:UTrade:Cash 3089.88 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 93 CSKO {32.59 USD} @ 33.31 USD Income:US:UTrade:CSKO:Gains 2014-12-01 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut10447525 Assets:US:UTrade:Cash 6.82 USD Income:US:UTrade:CSKO:Dividend -6.82 USD 2014-12-02 * "(BUY) BOUGHT +HOOL 55 @66.61" ^ut74330963 Assets:US:UTrade:Cash -3671.50 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 55 HOOL {66.61 USD} 2014-12-13 * "(SELL) SOLD +HOOL 50 @39.13 (LOT 42.33)" ^ut79580321 Assets:US:UTrade:Cash 1948.55 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 50 HOOL {42.33 USD} @ 39.13 USD Income:US:UTrade:HOOL:Gains 2014-12-20 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut60292945 Assets:US:UTrade:Cash 5.17 USD Income:US:UTrade:CSKO:Dividend -5.17 USD 2014-12-22 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut98071623 Assets:US:UTrade:Cash 19.97 USD Income:US:UTrade:HOOL:Dividend -19.97 USD 2014-12-31 * "(BUY) BOUGHT +MSFX 87 @50.34" ^ut94789086 Assets:US:UTrade:Cash -4387.53 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:MSFX 87 MSFX {50.34 USD} 2015-01-05 * "(DIV) ORDINARY DIVIDEND~CSKO" ^ut89525843 Assets:US:UTrade:Cash 23.67 USD Income:US:UTrade:CSKO:Dividend -23.67 USD 2015-01-17 * "(XFER) CLIENT REQUESTED ELECTRONIC FUNDING" ^ut92120597 Assets:US:UTrade:Cash 4876.00 USD Assets:US:BofA:Checking -4876.00 USD 2015-01-19 * "(BUY) BOUGHT +HOOL 40 @99.15" ^ut76302399 Assets:US:UTrade:Cash -3973.95 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 40 HOOL {99.15 USD} 2015-02-02 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut57564142 Assets:US:UTrade:Cash 7.98 USD Income:US:UTrade:HOOL:Dividend -7.98 USD 2015-02-06 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut90692243 Assets:US:UTrade:Cash 12.27 USD Income:US:UTrade:BAPL:Dividend -12.27 USD 2015-02-20 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut34606455 Assets:US:UTrade:Cash 14.83 USD Income:US:UTrade:BAPL:Dividend -14.83 USD 2015-02-22 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut16360724 Assets:US:UTrade:Cash 24.24 USD Income:US:UTrade:BAPL:Dividend -24.24 USD 2015-02-23 balance Assets:US:UTrade:Cash 1432.40 USD beangulp-0.2.0/examples/tests/utrade/UTrade20150720.csv000066400000000000000000000015421474340320100222740ustar00rootroot00000000000000DATE,TYPE,REF #,DESCRIPTION,FEES,AMOUNT,BALANCE 2015-05-17,DIV,57567122,ORDINARY DIVIDEND~HOOL,0,15.96,872.93 2015-05-18,SELL,98562913,SOLD +BAPL 146 @32.01 (LOT 26.15),7.95,4665.51,5538.44 2015-05-19,DIV,45119594,ORDINARY DIVIDEND~HOOL,0,13.91,5552.35 2015-05-26,SELL,53367649,SOLD +CSKO 89 @66.30 (LOT 63.85),7.95,5892.75,11445.10 2015-06-07,BUY,56695140,BOUGHT +HOOL 777 @5.16,7.95,-4017.27,7427.83 2015-06-19,DIV,17207133,ORDINARY DIVIDEND~BAPL,0,12.79,7440.62 2015-06-20,DIV,50840576,ORDINARY DIVIDEND~BAPL,0,13.89,7454.51 2015-06-25,DIV,88647893,ORDINARY DIVIDEND~BAPL,0,10.13,7464.64 2015-06-29,BUY,22740217,BOUGHT +CSKO 54 @64.93,7.95,-3514.17,3950.47 2015-07-02,DIV,27217440,ORDINARY DIVIDEND~HOOL,0,26.19,3976.66 2015-07-18,DIV,64873832,ORDINARY DIVIDEND~BAPL,0,9.80,3986.46 2015-07-19,BUY,51600680,BOUGHT +HOOL 31 @97.08,7.95,-3017.43,969.03 beangulp-0.2.0/examples/tests/utrade/UTrade20150720.csv.beancount000066400000000000000000000043301474340320100242470ustar00rootroot00000000000000;; Account: Assets:US:UTrade ;; Date: 2015-07-20 ;; Name: utrade.UTrade20150720.csv 2015-05-17 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut57567122 Assets:US:UTrade:Cash 15.96 USD Income:US:UTrade:HOOL:Dividend -15.96 USD 2015-05-18 * "(SELL) SOLD +BAPL 146 @32.01 (LOT 26.15)" ^ut98562913 Assets:US:UTrade:Cash 4665.51 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:BAPL 146 BAPL {26.15 USD} @ 32.01 USD Income:US:UTrade:BAPL:Gains 2015-05-19 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut45119594 Assets:US:UTrade:Cash 13.91 USD Income:US:UTrade:HOOL:Dividend -13.91 USD 2015-05-26 * "(SELL) SOLD +CSKO 89 @66.30 (LOT 63.85)" ^ut53367649 Assets:US:UTrade:Cash 5892.75 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 89 CSKO {63.85 USD} @ 66.30 USD Income:US:UTrade:CSKO:Gains 2015-06-07 * "(BUY) BOUGHT +HOOL 777 @5.16" ^ut56695140 Assets:US:UTrade:Cash -4017.27 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 777 HOOL {5.16 USD} 2015-06-19 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut17207133 Assets:US:UTrade:Cash 12.79 USD Income:US:UTrade:BAPL:Dividend -12.79 USD 2015-06-20 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut50840576 Assets:US:UTrade:Cash 13.89 USD Income:US:UTrade:BAPL:Dividend -13.89 USD 2015-06-25 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut88647893 Assets:US:UTrade:Cash 10.13 USD Income:US:UTrade:BAPL:Dividend -10.13 USD 2015-06-29 * "(BUY) BOUGHT +CSKO 54 @64.93" ^ut22740217 Assets:US:UTrade:Cash -3514.17 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:CSKO 54 CSKO {64.93 USD} 2015-07-02 * "(DIV) ORDINARY DIVIDEND~HOOL" ^ut27217440 Assets:US:UTrade:Cash 26.19 USD Income:US:UTrade:HOOL:Dividend -26.19 USD 2015-07-18 * "(DIV) ORDINARY DIVIDEND~BAPL" ^ut64873832 Assets:US:UTrade:Cash 9.80 USD Income:US:UTrade:BAPL:Dividend -9.80 USD 2015-07-19 * "(BUY) BOUGHT +HOOL 31 @97.08" ^ut51600680 Assets:US:UTrade:Cash -3017.43 USD Expenses:Financial:Fees 7.95 USD Assets:US:UTrade:HOOL 31 HOOL {97.08 USD} 2015-07-20 balance Assets:US:UTrade:Cash 969.03 USD beangulp-0.2.0/pyproject.toml000066400000000000000000000035101474340320100162170ustar00rootroot00000000000000[build-system] requires = ['setuptools >= 60.0'] build-backend = 'setuptools.build_meta' [project] name = 'beangulp' version = '0.2.0' license = { file = 'LICENSE' } description = 'Importers Framework for Beancount' readme = 'README.rst' authors = [ { name = 'Martin Blais', email = 'blais@furius.ca' }, { name = 'Daniele Nicolodi', email = 'daniele@grinta.net' }, ] maintainers = [ { name = 'Daniele Nicolodi', email = 'daniele@grinta.net' }, ] keywords = ['accounting', 'ledger', 'beancount', 'importer', 'import', 'converter', 'conversion'] classifiers = [ 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: Python :: 3.13', 'Topic :: Office/Business :: Financial :: Accounting', 'Topic :: Text Processing :: General', ] dependencies = [ 'beancount >=2.3.5', 'beautifulsoup4', 'chardet', 'click >8.0', 'lxml', 'python-magic >=0.4.12; sys_platform != "win32"', ] [project.urls] homepage = 'https://github.com/beancount/beangulp' issues = 'https://github.com/beancount/beangulp/issues' [tool.setuptools.packages] find = {} [tool.ruff] line-length = 128 target-version = 'py37' lint.select = ['E', 'F', 'W', 'UP', 'B', 'C4', 'PL', 'RUF'] lint.ignore = [ 'B007', 'B020', 'E731', 'PLR0911', 'PLR0912', 'PLR0913', 'PLR0915', 'PLR2004', 'PLW2901', 'RUF012', 'UP015', 'UP032', ] [tool.pytest.ini_options] minversion = '6.0' addopts = ['--doctest-glob=*.rst'] doctest_optionflags = ['ELLIPSIS', 'NORMALIZE_WHITESPACE'] testpaths = ['beangulp'] beangulp-0.2.0/requirements.txt000066400000000000000000000001551474340320100165710ustar00rootroot00000000000000beancount >=2.3.5 beautifulsoup4 chardet click >7.0 lxml python-magic >=0.4.12; sys_platform != 'win32' petl beangulp-0.2.0/setup.py000066400000000000000000000000451474340320100150150ustar00rootroot00000000000000import setuptools setuptools.setup() beangulp-0.2.0/tools/000077500000000000000000000000001474340320100144445ustar00rootroot00000000000000beangulp-0.2.0/tools/migrate_files.py000077500000000000000000000042001474340320100176270ustar00rootroot00000000000000#!/usr/bin/env python3 """Migrate old test files to the new one. This script can be used to convert old test files for beancount.ingest to the newer scheme where a single expected file is used. For example, this converts: 2015-06-13.ofx.qbo 2015-06-13.ofx.qbo.extract 2015-06-13.ofx.qbo.file_account 2015-06-13.ofx.qbo.file_date 2015-06-13.ofx.qbo.file_name to: 2015-06-13.ofx.qbo 2015-06-13.ofx.qbo.beancount """ import functools import os import re from os import path from typing import List import click def read_or_empty(filename: str) -> str: if path.exists(filename): with open(filename) as infile: return infile.read().rstrip() else: return '' def process_files(filename: str): """Rename files around the given absolute filename.""" account = read_or_empty(filename + ".file_account") date = read_or_empty(filename + ".file_date") name = read_or_empty(filename + ".file_name") extract = read_or_empty(filename + ".extract") with open(filename + ".beancount", "w") as outfile: pr = functools.partial(print, file=outfile) pr(';; Account: {}'.format(account or'')) pr(';; Date: {}'.format(date or '')) pr(';; Name: {}'.format(name or '')) if extract: pr(extract) for ext in [".file_account", ".file_date", ".file_name", ".extract"]: if path.exists(filename + ext): os.remove(filename + ext) @click.command() @click.argument('directories',type=click.Path(exists=True, resolve_path=True), nargs=-1) def main(directories: List[str]): for directory in directories: for root, dirs, files in os.walk(directory): for filename in sorted(files): afilename = path.join(root, filename) if re.search(r"\.(py|beancount)$", afilename): continue if re.search(r"\.(extract|file_account|file_name|file_date)", afilename): continue if path.exists(afilename + ".beancount"): continue process_files(afilename) if __name__ == '__main__': main()