pax_global_header00006660000000000000000000000064147513404450014521gustar00rootroot0000000000000052 comment=22c3a23e44c8463634e7dc22fc7e9981a70b0673 beanprice-2.0.0/000077500000000000000000000000001475134044500134505ustar00rootroot00000000000000beanprice-2.0.0/.editorconfig000066400000000000000000000003271475134044500161270ustar00rootroot00000000000000# Editor configuration, see http://editorconfig.org root = true [*] charset = utf-8 indent_style = space insert_final_newline = true trim_trailing_whitespace = true [*.py] indent_size = 4 [*.yml] indent_size = 2 beanprice-2.0.0/.github/000077500000000000000000000000001475134044500150105ustar00rootroot00000000000000beanprice-2.0.0/.github/workflows/000077500000000000000000000000001475134044500170455ustar00rootroot00000000000000beanprice-2.0.0/.github/workflows/beancount.yml000066400000000000000000000006051475134044500215470ustar00rootroot00000000000000name: beancount on: [push, pull_request] jobs: test: runs-on: ubuntu-latest strategy: fail-fast: false steps: - uses: actions/checkout@v2 - uses: actions/setup-python@v2 with: python-version: '3.9' - run: pip install -r requirements_dev.txt - run: pylint beanprice - run: pytest beanprice - run: mypy beanprice beanprice-2.0.0/.gitignore000066400000000000000000000001601475134044500154350ustar00rootroot00000000000000*.DS_Store *.a *.pyc *~ .cache .idea .noseids .pytest_cache TAGS venv build dist beanprice.egg-info __pycache__ beanprice-2.0.0/.pylintrc000066400000000000000000000233151475134044500153210ustar00rootroot00000000000000[MASTER] # Specify a configuration file. #rcfile= # Python code to execute, usually for sys.path manipulation such as # pygtk.require(). #init-hook= # Add files or directories to the blacklist. They should be base names, not # paths. ignore=.git,BUILD # Pickle collected data for later comparisons. persistent=yes # List of plugins (as comma separated values of python modules names) to load, # usually to register additional checkers. load-plugins= # DEPRECATED #include-ids=no # DEPRECATED #symbols=no [REPORTS] # Set the output format. Available formats are text, parseable, colorized, msvs # (visual studio) and html. You can also give a reporter class, eg # mypackage.mymodule.MyReporterClass. output-format=text # Tells whether to display a full report or only the messages reports=no # Python expression which should return a note less than 10 (10 is the highest # note). You have access to the variables errors warning, statement which # respectively contain the number of errors / warnings messages and the total # number of statements analyzed. This is used by the global evaluation report # (RP0004). evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) # Template used to display messages. This is a python new-style format string # used to format the message information. See doc for all details msg-template="{path}:{line}: [{msg_id}({symbol}), {obj}] {msg}" [MESSAGES CONTROL] # Enable the message, report, category or checker with the given id(s). You can # either give multiple identifier separated by comma (,) or put this option # multiple time. See also the "--disable" option for examples. enable=all # Disable the message, report, category or checker with the given id(s). You # can either give multiple identifiers separated by comma (,) or put this # option multiple times (only on the command line, not in the configuration # file where it should appear only once).You can also use "--disable=all" to # disable everything first and then reenable specific checks. For example, if # you want to run only the similarities checker, you can use "--disable=all # --enable=similarities". If you want to run only the classes checker, but have # no Warning level messages displayed, use"--disable=all --enable=classes # --disable=W" disable=locally-disabled, suppressed-message, missing-docstring, too-many-lines, multiple-statements, superfluous-parens, ungrouped-imports, wrong-import-position, no-self-argument, no-member, no-value-for-parameter, too-many-function-args, unsubscriptable-object, too-many-nested-blocks, duplicate-code, too-few-public-methods, too-many-public-methods, too-many-branches, too-many-arguments, too-many-locals, too-many-statements, attribute-defined-outside-init, protected-access, arguments-differ, abstract-method, fixme, global-variable-undefined, global-statement, unused-variable, unused-argument, redefined-outer-name, redefined-builtin, undefined-loop-variable, broad-except, logging-format-interpolation, anomalous-backslash-in-string, len-as-condition, no-else-return, invalid-unary-operand-type, no-name-in-module, inconsistent-return-statements, not-callable, stop-iteration-return, assignment-from-no-return, c-extension-no-member, cyclic-import, isinstance-second-argument-not-valid-type, missing-timeout, consider-using-f-string, consider-using-with, use-implicit-booleaness-not-comparison-to-string, use-implicit-booleaness-not-comparison-to-zero, too-many-positional-arguments, possibly-used-before-assignment, arguments-renamed # Notes: # bad-continuation: Is buggy, see https://github.com/PyCQA/pylint/issues/3512 [VARIABLES] # Tells whether we should check for unused import in __init__ files. init-import=no # A regular expression matching the name of dummy variables (i.e. expectedly # not used). dummy-variables-rgx=_$|dummy # List of additional names supposed to be defined in builtins. Remember that # you should avoid to define new builtins when possible. additional-builtins= [TYPECHECK] # Tells whether missing members accessed in mixin class should be ignored. A # mixin class is detected if its name ends with "mixin" (case insensitive). ignore-mixin-members=yes # List of module names for which member attributes should not be checked # (useful for modules/projects where namespaces are manipulated during runtime # and thus existing member attributes cannot be deduced by static analysis ignored-modules= # List of classes names for which member attributes should not be checked # (useful for classes with attributes dynamically set). ignored-classes= # List of members which are set dynamically and missed by pylint inference # system, and so shouldn't trigger E0201 when accessed. Python regular # expressions are accepted. #generated-members=REQUEST,acl_users,aq_parent generated-members= [LOGGING] # Logging modules to check that the string format arguments are in logging # function parameter format logging-modules=logging [MISCELLANEOUS] # List of note tags to take in consideration, separated by a comma. notes=FIXME,XXX,TODO [BASIC] # Good variable names which should always be accepted, separated by a comma good-names=f,i,j,k,ex,Run,_ # Bad variable names which should always be refused, separated by a comma bad-names=foo,bar,baz,toto,tutu,tata # Colon-delimited sets of names that determine each other's naming style when # the name regexes allow several styles. name-group= # Include a hint for the correct naming format with invalid-name include-naming-hint=no # Regular expression matching correct module names module-rgx=(([a-z_][a-z0-9_\-]*)|([A-Z][a-zA-Z0-9]+))$ # Regular expression matching correct method names method-rgx=[a-z_][a-zA-Z0-9_]{2,72}$ # Regular expression matching correct class names class-rgx=[A-Z_][a-zA-Z0-9]+$ # Regular expression matching correct class attribute names class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ # Regular expression matching correct inline iteration names inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ # Regular expression matching correct variable names variable-rgx=(_?[a-z_][a-z0-9_]{2,30}|__|mu|no)$ # Regular expression matching correct constant names const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ # Regular expression matching correct attribute names attr-rgx=[a-z_][a-z0-9_]{2,30}$ # Regular expression matching correct argument names argument-rgx=(_?[a-z_][a-z0-9_]{2,30}|__|mu)$ # Regular expression matching correct function names function-rgx=_?[a-z_][a-zA-Z0-9_]{2,64}$ # Regular expression which should only match function or class names that do # not require a docstring. no-docstring-rgx=__.*__ # Minimum line length for functions/classes that require docstrings, shorter # ones are exempt. docstring-min-length=-1 [FORMAT] # Maximum number of characters on a single line. max-line-length=92 # Regexp for a line that is allowed to be longer than the limit. ignore-long-lines=^\s*(# )??$ # Allow the body of an if to be on the same line as the test if there is no # else. single-line-if-stmt=no # Maximum number of lines in a module max-module-lines=1000 # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 # tab). indent-string=' ' # Number of spaces of indent required inside a hanging or continued line. indent-after-paren=4 [SIMILARITIES] # Minimum lines number of a similarity. min-similarity-lines=4 # Ignore comments when computing similarities. ignore-comments=yes # Ignore docstrings when computing similarities. ignore-docstrings=yes # Ignore imports when computing similarities. ignore-imports=no [DESIGN] # Maximum number of arguments for function / method max-args=5 # Argument names that match this expression will be ignored. Default to name # with leading underscore ignored-argument-names=_.* # Maximum number of locals for function / method body max-locals=20 # Maximum number of return / yield for function / method body max-returns=6 # Maximum number of branch for function / method body max-branches=12 # Maximum number of statements in function / method body max-statements=50 # Maximum number of parents for a class (see R0901). max-parents=7 # Maximum number of attributes for a class (see R0902). max-attributes=7 # Minimum number of public methods for a class (see R0903). min-public-methods=2 # Maximum number of public methods for a class (see R0904). max-public-methods=20 [IMPORTS] # Deprecated modules which should not be used, separated by a comma deprecated-modules=stringprep,optparse # Create a graph of every (i.e. internal and external) dependencies in the # given file (report RP0402 must not be disabled) import-graph= # Create a graph of external dependencies in the given file (report RP0402 must # not be disabled) ext-import-graph= # Create a graph of internal dependencies in the given file (report RP0402 must # not be disabled) int-import-graph= [CLASSES] # List of interface methods to ignore, separated by a comma. This is used for # instance to not check methods defines in Zope's Interface base class. ## ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by # List of method names used to declare (i.e. assign) instance attributes. defining-attr-methods=__init__,__new__,setUp # List of valid names for the first argument in a class method. valid-classmethod-first-arg=cls # List of valid names for the first argument in a metaclass class method. valid-metaclass-classmethod-first-arg=mcs [EXCEPTIONS] # Exceptions that will emit a warning when being caught. Defaults to # "Exception" overgeneral-exceptions=builtins.Exception beanprice-2.0.0/COPYING000066400000000000000000000436471475134044500145210ustar00rootroot00000000000000 GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Lesser General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Lesser General Public License instead of this License. ------------------------------------------------------------ Note: This is a GNU GPL "v2 only" license. This is not a GNU GPL "v2 or any later version" license. ---Martin Blais beanprice-2.0.0/README.md000066400000000000000000000125361475134044500147360ustar00rootroot00000000000000# beanprice: Price quotes fetcher for Beancount ## Description A script to fetch market data prices from various sources on the internet and render them for plain text accounting price syntax (and Beancount). This used to be located within Beancount itself (at v2) under beancount.prices. This repo will contain all future updates to that script and to those price sources. ## Documentation Some documentation is still part of Beancount. More about how to use this can be found on that [mailing-list](https://groups.google.com/forum/#!forum/beancount). Otherwise read the source. ## Quick start To install beanprice, run: ```shell pip install git+https://github.com/beancount/beanprice.git ``` You can fetch the latest price of a stock by running: ```shell bean-price -e 'USD:yahoo/AAPL' ``` To fetch the latest prices from your beancount file, first ensure that commodities have price metadata, e.g. ``` 2000-01-01 commodity AAPL price: "USD:yahoo/AAPL" ``` Then run: ```shell bean-price ledger.beancount ``` To update prices up to the present day, run: ```shell bean-price --update ledger.beancount ``` For more detailed guide for price fetching, read . ## Price source info The following price sources are available: | Name | Module | Provides prices for | Base currency | Latest price? | Historical price? | |-------------------------|---------------------------|-----------------------------------------------------------------------------------|----------------------------------------------------------------------------------|---------------|-------------------| | Alphavantage | `beanprice.alphavantage` | [Stocks, FX, Crypto](http://alphavantage.co) | Many currencies | ✓ | ✕ | | Coinbase | `beanprice.coinbase` | [Most common (crypto)currencies](https://api.coinbase.com/v2/exchange-rates) | [Many currencies](https://api.coinbase.com/v2/currencies) | ✓ | ✓ | | Coincap | `beanprice.coincap` | [Most common (crypto)currencies](https://docs.coincap.io) | USD | ✓ | ✓ | | Coinmarketcap | `beanprice.coinmarketcap` | [Most common (crypto)currencies](https://coinmarketcap.com/api/documentation/v1/) | Many Currencies | ✓ | ✕ | | European Central Bank API| `beanprice.ecbrates` | [Many currencies](https://data.ecb.europa.eu/search-results?searchTerm=exchange%20rates) | [Many currencies](https://data.ecb.europa.eu/search-results?searchTerm=exchange%20rates) (Derived from EUR rates)| ✓ | ✓ | | IEX | `beanprice.iex` | [Trading symbols](https://iextrading.com/trading/eligible-symbols/) | USD | ✓ | 🚧 (Not yet!) | | OANDA | `beanprice.oanda` | [Many currencies](https://developer.oanda.com/exchange-rates-api/v1/currencies/) | [Many currencies](https://developer.oanda.com/exchange-rates-api/v1/currencies/) | ✓ | ✓ | | Quandl | `beanprice.quandl` | [Various datasets](https://www.quandl.com/search) | [Various datasets](https://www.quandl.com/search) | ✓ | ✓ | | Rates API | `beanprice.ratesapi` | [Many currencies](https://api.exchangerate.host/symbols) | [Many currencies](https://api.exchangerate.host/symbols) | ✓ | ✓ | | Thrift Savings Plan | `beanprice.tsp` | TSP Funds | USD | ✓ | ✓ | | Yahoo | `beanprice.yahoo` | Many currencies | Many currencies | ✓ | ✓ | | EastMoneyFund(天天基金) | `beanprice.eastmoneyfund` | [Chinese Funds](http://fund.eastmoney.com/js/fundcode_search.js) | CNY | ✓ | ✓ | More price sources can be found at [awesome-beancount.com](https://awesome-beancount.com/#price-sources) website. ## Testing Run tests: ``` pytest beanprice ``` Lint: ``` pylint beanprice ``` Type checker: ``` mypy beanprice ``` ## Copyright and License Copyright (C) 2007-2020 Martin Blais. All Rights Reserved. This code is distributed under the terms of the "GNU GPLv2 only". See COPYING file for details. beanprice-2.0.0/beanprice/000077500000000000000000000000001475134044500154005ustar00rootroot00000000000000beanprice-2.0.0/beanprice/BUILD000066400000000000000000000031551475134044500161660ustar00rootroot00000000000000package(default_visibility = ["//visibility:public"]) py_library( name = "__init__", srcs = ["__init__.py"], ) py_library( name = "net_utils", srcs = ["net_utils.py"], ) py_test( name = "net_utils_test", srcs = ["net_utils_test.py"], deps = [ ":net_utils", ], ) py_library( name = "price", srcs = ["price.py"], deps = [ ":date_utils", "//beancount/core:amount", "//beancount/core:data", "//beancount/core:number", "//beancount/core:prices", "//beancount/core:getters", "//beancount/parser:printer", "//beancount/parser:version", "//beancount/ops:find_prices", "//beancount/ops:lifetimes", "//beancount/prices:__init__", "//beancount:loader", ], ) # Again, the problem here is that this code calls the binary. py_test( name = "price_test", srcs = ["price_test.py"], deps = [ "//beancount/core:number", "//beancount:loader", "//beancount/parser:cmptest", "//beancount/ops:find_prices", "//beancount/prices:price", "//beancount/prices:source", "//beancount/prices/sources:oanda", "//beancount/prices/sources:yahoo", "//beancount/utils:test_utils", "//beancount:plugins_for_tests", ], ) py_library( name = "source", srcs = ["source.py"], deps = [ "//beancount/core:number", ], ) py_library( name = "date_utils", srcs = ["date_utils.py"], ) py_test( name = "date_utils_test", srcs = ["date_utils_test.py"], deps = [ ":date_utils", ], ) beanprice-2.0.0/beanprice/__init__.py000066400000000000000000000125041475134044500175130ustar00rootroot00000000000000"""Fetch prices from the internet and output them as Beancount price directives. This script accepts a list of Beancount input filenames, and fetches prices required to compute market values for current positions: bean-price /home/joe/finances/joe.beancount The list of fetching jobs to carry out is derived automatically from the input file (see section below for full details). It is also possible to provide a list of specific price fetching jobs to run, e.g., bean-price -e google/TSE:XUS yahoo/AAPL mysources.morningstar/RBF1005 The general format of each of these "source strings" is /[^] The "module" is the name of a Python module that contains a Source class which can be instantiated and connect to a data source to extract price data. These modules are automatically imported by name and instantiated in order to pull the price from a particular data source. This allows you to write your own supplementary fetcher codes without having to modify this script. Default implementations are provided to provide access to prices from Yahoo! Finance or Google Finance, which cover a large universe of common public investment types (e.g. stock tickers). As a convenience, the module name is always first searched under the "beanprice.sources" package, where those default source implementations live. This is how, for example, in order to use the provided Google Finance data fetcher you don't have to write "beanprice.sources.yahoo/AAPL" but simply "yahoo/AAPL". Date ---- By default, this script will fetch prices at the latest available date & time. You can use an option to fetch historical prices for a desired date instead: bean-price --date=2015-02-03 Inverse ------- Sometimes, prices are available for the inverse of an instrument. This is often the case for currencies. For example, the price of "CAD" in USD" is provided by the USD/CAD market, which gives the price of a US dollar in Canadian dollars. In order specify this, you can prepend "^" to the instrument to instruct the driver to compute the inverse of the given price: bean-price -e USD:google/^CURRENCY:USDCAD If a source price is to be inverted, like this, the precision could be different than what is fetched. For instance, if the price of USD/CAD is 1.32759, it would output be this from the above directive: 2015-10-28 price CAD 0.753244601119 USD By default, inverted rates will be rounded similarly to how other Price directives were rounding those numbers. Swap Inverted ------------- If you prefer to have the output Price entries with swapped currencies instead of inverting the rate itself, you can use the --swap-inverted option. In the previous example for the price of CAD, it would output this: 2015-10-28 price USD 1.32759 CAD This works since the Beancount price database computes and interpolates the reciprocals automatically for all pairs of commodities in its database. Prices Needed for a Beancount File ---------------------------------- You can also provide a filename to extract the list of tickers to fetch from a Beancount input file, e.g.: bean-price /home/joe/finances/joe.beancount There are many ways to extract a list of commodities with needed prices from a Beancount input file: - Prices for all the holdings that were seen held-at-cost at a particular date. - Prices for holdings held at a particular date which were price converted from some other commodity in the past (i.e., for currencies). - The list of all Commodity directives present in the file. For each of those holdings, the corresponding Commodity directive is consulted and its "price" metadata field is used to specify where to attempt to fetch prices. You should have directives like this in your input file: 2007-07-20 commodity VEA price: "google/NYSEARCA:VEA" The "price" metadata can be a comma-separated list of sources to try out, in which case each of the sources will be looked at : 2007-07-20 commodity VEA price: "google/CURRENCY:USDCAD,yahoo/USDCAD" - Existing price directives for the same data are excluded by default, since the price is already in the file. By default, the list of tickers to be fetched includes only the intersection of these lists. The general intent of the user of this script is to fetch missing prices, and only needed ones, for a particular date. * Use the --date option to change the applied date. * Use the --all option to fetch the entire set of prices, regardless of holdings and date. * Use --clobber to ignore existing price directives. You can also print the list of prices to be fetched with the --dry-run option, which stops short of actually fetching the missing prices (it just prints the list of fetches it would otherwise attempt). Caching ------- Prices are automatically cached. You can disable the cache with an option: bean-price --no-cache You can also instruct the script to clear the cache before fetching its prices: bean-price --clear-cache About Sources and Data Availability ----------------------------------- IMPORTANT: Note that each source may support a different routine for getting its latest data and for fetching historical/dated data, and that each of these may differ in their support. For example, Google Finance does not support fetching historical data for its CURRENCY:* instruments. """ __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" beanprice-2.0.0/beanprice/date_utils.py000066400000000000000000000024411475134044500201100ustar00rootroot00000000000000"""Date utilities.""" __copyright__ = "Copyright (C) 2020 Martin Blais" __license__ = "GNU GPLv2" import contextlib import os import time import dateutil.parser def parse_date_liberally(string, parse_kwargs_dict=None): """Parse arbitrary strings to dates. This function is intended to support liberal inputs, so that we can use it in accepting user-specified dates on command-line scripts. Args: string: A string to parse. parse_kwargs_dict: Dict of kwargs to pass to dateutil parser. Returns: A datetime.date object. """ # At the moment, rely on the most excellent dateutil. if parse_kwargs_dict is None: parse_kwargs_dict = {} return dateutil.parser.parse(string, **parse_kwargs_dict).date() @contextlib.contextmanager def intimezone(tz_value: str): """Temporarily reset the value of TZ. This is used for testing. Args: tz_value: The value of TZ to set for the duration of this context. Returns: A contextmanager in the given timezone locale. """ tz_old = os.environ.get("TZ", None) os.environ["TZ"] = tz_value time.tzset() try: yield finally: if tz_old is None: del os.environ["TZ"] else: os.environ["TZ"] = tz_old time.tzset() beanprice-2.0.0/beanprice/date_utils_test.py000066400000000000000000000027431475134044500211540ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2020 Martin Blais" __license__ = "GNU GPLv2" import unittest import datetime import dateutil from beanprice import date_utils class TestDateUtils(unittest.TestCase): def test_parse_date_liberally(self): const_date = datetime.date(2014, 12, 7) test_cases = ( ("12/7/2014",), ("7-Dec-2014",), ("7/12/2014", {"parserinfo": dateutil.parser.parserinfo(dayfirst=True)}), ("12/7", {"default": datetime.datetime(2014, 1, 1)}), ("7.12.2014", {"dayfirst": True}), ("14 12 7", {"yearfirst": True}), ("Transaction of 7th December 2014", {"fuzzy": True}), ) for case in test_cases: if len(case) == 2: parse_date = date_utils.parse_date_liberally(case[0], case[1]) else: parse_date = date_utils.parse_date_liberally(case[0]) self.assertEqual(const_date, parse_date) def test_intimezone(self): with date_utils.intimezone("America/New_York"): now_nyc = datetime.datetime.now() with date_utils.intimezone("Europe/Berlin"): now_berlin = datetime.datetime.now() with date_utils.intimezone("Asia/Tokyo"): now_tokyo = datetime.datetime.now() self.assertNotEqual(now_nyc, now_berlin) self.assertNotEqual(now_berlin, now_tokyo) self.assertNotEqual(now_tokyo, now_nyc) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/net_utils.py000066400000000000000000000015741475134044500177670ustar00rootroot00000000000000"""Network utilities.""" __copyright__ = "Copyright (C) 2015-2016 Martin Blais" __license__ = "GNU GPLv2" import logging from urllib import request from urllib import error def retrying_urlopen(url, timeout=5, max_retry=5): """Open and download the given URL, retrying if it times out. Args: url: A string, the URL to fetch. timeout: A timeout after which to stop waiting for a response and return an error. max_retry: The maximum number of times to retry. Returns: The contents of the fetched URL. """ for _ in range(max_retry): logging.debug("Reading %s", url) try: response = request.urlopen(url, timeout=timeout) if response: break except error.URLError: return None if response and response.getcode() != 200: return None return response beanprice-2.0.0/beanprice/net_utils_test.py000066400000000000000000000024701475134044500210220ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2015-2016 Martin Blais" __license__ = "GNU GPLv2" import http.client import unittest from unittest import mock from beanprice import net_utils class TestRetryingUrlopen(unittest.TestCase): def test_success_200(self): response = http.client.HTTPResponse(mock.MagicMock()) response.status = 200 with mock.patch("urllib.request.urlopen", return_value=response): self.assertIs(net_utils.retrying_urlopen("http://nowhere.com"), response) def test_success_other(self): response = http.client.HTTPResponse(mock.MagicMock()) with mock.patch("urllib.request.urlopen", return_value=response): self.assertIsNone(net_utils.retrying_urlopen("http://nowhere.com")) def test_timeout_once(self): response = http.client.HTTPResponse(mock.MagicMock()) response.status = 200 with mock.patch("urllib.request.urlopen", side_effect=[None, response]): self.assertIs(net_utils.retrying_urlopen("http://nowhere.com"), response) def test_max_retry(self): with mock.patch( "urllib.request.urlopen", side_effect=[None, None, None, None, None, None] ): self.assertIsNone(net_utils.retrying_urlopen("http://nowhere.com")) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/price.py000066400000000000000000001036461475134044500170660ustar00rootroot00000000000000"""Driver code for the price script.""" __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" import argparse import collections import datetime import functools from os import path import shelve import tempfile import hashlib import re import sys import logging from concurrent import futures from typing import Any, Dict, List, Optional, NamedTuple, Tuple from dateutil import tz from beancount.core.number import ONE from beancount import loader from beancount.core import data from beancount.core import amount from beancount.core import prices from beancount.core import getters from beancount.ops import lifetimes from beancount.parser import printer from beancount.ops import find_prices from beanprice import date_utils import beanprice # A price source. # # module: A Python module, the module to be called to create a price source. # symbol: A ticker symbol in the universe of the source. # invert: A boolean, true if we need to invert the currency. class PriceSource(NamedTuple): module: Any symbol: str invert: bool # A dated price source description. # # Attributes: # base: A commodity string, the base for the given symbol from the input file. # This may be null if we don't have a mapping for it. # quote: A commodity string, the quote currency that defines the units of the price. # This is also intended to be a commodity from the input file, and similarly, # may be null. # date: A datetime.date object for the date to be fetched, or None # with the meaning of fetching the latest price. # sources: A list of PriceSource instances describing where to fetch prices from. class DatedPrice(NamedTuple): base: Optional[str] quote: Optional[str] date: Optional[datetime.date] sources: List[PriceSource] # The Python package where the default sources are found. DEFAULT_PACKAGE = "beanprice.sources" # Stand-in currency name for unknown currencies. UNKNOWN_CURRENCY = "?" # A cache for the prices. _CACHE = None # Expiration for latest prices in the cache. DEFAULT_EXPIRATION = datetime.timedelta(seconds=30 * 60) # 30 mins. # The default source parser is back. DEFAULT_SOURCE = "beanprice.sources.yahoo" def format_dated_price_str(dprice: DatedPrice) -> str: """Convert a dated price to a one-line printable string. Args: dprice: A DatedPrice instance. Returns: The string for a DatedPrice instance. """ psstrs = [ "{}({}{})".format( psource.module.__name__, "1/" if psource.invert else "", psource.symbol ) for psource in dprice.sources ] base_quote = "{} /{}".format(dprice.base, dprice.quote) return "{:<32} @ {:10} [ {} ]".format( base_quote, dprice.date.isoformat() if dprice.date else "latest", ",".join(psstrs) ) def parse_source_map(source_map_spec: str) -> Dict[str, List[PriceSource]]: """Parse a source map specification string. Source map specifications allow the specification of multiple sources for multiple quote currencies and follow the following syntax: :,,... :,... Where a itself follows: /[^] The is resolved against the Python path, but first looked up under the package where the default price extractors lie. The presence of a '^' character indicates that we should use the inverse of the rate pull from this source. For example, for prices of AAPL in USD: USD:google/NASDAQ:AAPL,yahoo/AAPL Or for the exchange rate of a currency, such as INR in USD or in CAD: USD:google/^CURRENCY:USDINR CAD:google/^CURRENCY:CADINR Args: source_map_spec: A string, a full source map specification to be parsed. Returns: A dict of quote currency to price sources for that currency. Raises: ValueError: If an invalid pattern has been specified. """ source_map: Dict[str, List[PriceSource]] = collections.defaultdict(list) for source_list_spec in re.split("[ ;]", source_map_spec): match = re.match("({}):(.*)$".format(amount.CURRENCY_RE), source_list_spec) if not match: raise ValueError('Invalid source map pattern: "{}"'.format(source_list_spec)) currency, source_strs = match.groups() source_map[currency].extend( parse_single_source(source_str) for source_str in source_strs.split(",") ) return source_map def parse_single_source(source: str) -> PriceSource: """Parse a single source string. Source specifications follow the syntax: /[^] The is resolved against the Python path, but first looked up under the package where the default price extractors lie. Args: source: A single source string specification. Returns: A PriceSource tuple. Raises: ValueError: If invalid. """ match = re.match(r"([a-zA-Z]+[a-zA-Z0-9\._]+)/(\^?)([a-zA-Z0-9:=_\-\.\(\)]+)$", source) if not match: raise ValueError('Invalid source name: "{}"'.format(source)) short_module_name, invert, symbol = match.groups() module = import_source(short_module_name) return PriceSource(module, symbol, bool(invert)) def import_source(module_name: str): """Import the source module defined by the given name. The default location is handled here. Args: short_module_name: A string, the name of a Python module, which may be within the default package or a full name. Returns: A corresponding Python module object. Raises: ImportError: If the module cannot be imported. """ default_name = "{}.{}".format(DEFAULT_PACKAGE, module_name) try: __import__(default_name) return sys.modules[default_name] except ImportError: try: __import__(module_name) return sys.modules[module_name] except ImportError as exc: raise ImportError( 'Could not find price source module "{}"'.format(module_name) ) from exc def find_currencies_declared( entries: data.Entries, date: Optional[datetime.date] = None, ) -> List[Tuple[str, str, List[PriceSource]]]: """Return currencies declared in Commodity directives. If a 'price' metadata field is provided, include all the quote currencies there-in. Otherwise, the Commodity directive is ignored. Args: entries: A list of directives. date: A datetime.date instance. Returns: A list of (base, quote, list of PriceSource) currencies. The list of (base, quote) pairs is guaranteed to be unique. """ currencies = [] for entry in entries: if not isinstance(entry, data.Commodity): continue if date and entry.date >= date: break # Here we have to infer which quote currencies the commodity is for # (maybe down the road this should be better handled by providing a list # of quote currencies in the Commodity directive itself). # # First, we look for a "price" metadata field, which defines conversions # for various currencies. Each of these quote currencies generates a # pair in the output. source_str = entry.meta.get("price", None) if source_str is not None: if source_str == "": logging.debug( "Skipping ignored currency (with empty price): %s", entry.currency ) continue try: source_map = parse_source_map(source_str) except ValueError as exc: logging.warning( "Ignoring currency with invalid 'price' source: %s (%s)", entry.currency, exc, ) else: for quote, psources in source_map.items(): currencies.append((entry.currency, quote, psources)) else: # Otherwise we simply ignore the declaration. That is, a Commodity # directive without any "price" metadata would not register as a # declared currency. logging.debug("Ignoring currency with no metadata: %s", entry.currency) return currencies def log_currency_list(message, currencies): """Log a list of currencies to debug output. Args: message: A message string to prepend. currencies: A list of (base, quote) currency pair. """ logging.debug("-------- {}:".format(message)) for base, quote in currencies: logging.debug(" {:>32}".format("{} /{}".format(base, quote))) def get_price_jobs_at_date( entries: data.Entries, date: Optional[datetime.date] = None, inactive: bool = False, undeclared_source: Optional[str] = None, ): """Get a list of prices to fetch from a stream of entries. The active holdings held on the given date are included. Args: entries: A list of beancount entries, the name of a file to process. date: A datetime.date instance. inactive: Include currencies with no balance at the given date. The default is to only include those currencies which have a non-zero balance. undeclared_source: A string, the name of the default source module to use to pull prices for commodities without a price source metadata on their Commodity directive declaration. Returns: A list of DatedPrice instances. """ # Find the list of declared currencies, and from it build a mapping for # tickers for each (base, quote) pair. This is the only place tickers # appear. declared_triples = find_currencies_declared(entries, date) currency_map = {(base, quote): psources for base, quote, psources in declared_triples} # Compute the initial list of currencies to consider. if undeclared_source: # Use the full set of possible currencies. cur_at_cost = find_prices.find_currencies_at_cost(entries) cur_converted = find_prices.find_currencies_converted(entries, date) cur_priced = find_prices.find_currencies_priced(entries, date) currencies = cur_at_cost | cur_converted | cur_priced log_currency_list("Currency held at cost", cur_at_cost) log_currency_list("Currency converted", cur_converted) log_currency_list("Currency priced", cur_priced) default_source = import_source(undeclared_source) else: # Use the currencies from the Commodity directives. currencies = set(currency_map.keys()) default_source = None log_currency_list("Currencies in primary list", currencies) # By default, restrict to only the currencies with non-zero balances at the # given date. if not inactive: balance_currencies = find_prices.find_balance_currencies(entries, date) log_currency_list("Currencies held in assets", balance_currencies) currencies = currencies & balance_currencies log_currency_list("Currencies to fetch", currencies) # Build up the list of jobs to fetch prices for. jobs = [] for base_quote in currencies: psources = currency_map.get(base_quote, None) base, quote = base_quote # If there are no sources, create a default one. if not psources: psources = [PriceSource(default_source, base, False)] jobs.append(DatedPrice(base, quote, date, psources)) return sorted(jobs) # TODO(blais): This could be modified to use the get_daily_prices() interface, # or perhaps to extend it to intervals, and let the price source decide for # itself how to implement fetching (e.g., use a single call + filter, or use # multiple calls). Querying independently for each day is not the best strategy. def get_price_jobs_up_to_date( entries, date_last=None, inactive=False, undeclared_source=None, update_rate="weekday", compress_days=1, ): """Get a list of trailing prices to fetch from a stream of entries. The list of dates runs from the latest available price up to the latest date. Args: entries: list of Beancount entries date_last: The date up to where to find prices to as an exclusive range end. inactive: Include currencies with no balance at the given date. The default is to only include those currencies which have a non-zero balance. undeclared_source: A string, the name of the default source module to use to pull prices for commodities without a price source metadata on their Commodity directive declaration. Returns: A list of DatedPrice instances. """ price_map = prices.build_price_map(entries) # Find the list of declared currencies, and from it build a mapping for # tickers for each (base, quote) pair. This is the only place tickers # appear. declared_triples = find_currencies_declared(entries, date_last) currency_map = {(base, quote): psources for base, quote, psources in declared_triples} # Compute the initial list of currencies to consider. if undeclared_source: # Use the full set of possible currencies. cur_at_cost = find_prices.find_currencies_at_cost(entries) cur_converted = find_prices.find_currencies_converted(entries, date_last) cur_priced = find_prices.find_currencies_priced(entries, date_last) currencies = cur_at_cost | cur_converted | cur_priced log_currency_list("Currency held at cost", cur_at_cost) log_currency_list("Currency converted", cur_converted) log_currency_list("Currency priced", cur_priced) default_source = import_source(undeclared_source) else: # Use the currencies from the Commodity directives. currencies = set(currency_map.keys()) default_source = None log_currency_list("Currencies in primary list", currencies) # By default, restrict to only the currencies with non-zero balances # up to the given date. # Also, find the earliest start date to fetch prices from. # Look at both latest prices and start dates. lifetimes_map = lifetimes.get_commodity_lifetimes(entries) commodity_map = getters.get_commodity_directives(entries) if inactive: for base_quote in currencies: if lifetimes_map[base_quote]: # Use first date from lifetime lifetimes_map[base_quote] = [(lifetimes_map[base_quote][0][0], None)] else: # Insert never active commodities into lifetimes # Start from date of currency directive base, _ = base_quote commodity_entry = commodity_map.get(base, None) lifetimes_map[base_quote] = [(commodity_entry.date, None)] else: # Compress any lifetimes based on compress_days lifetimes_map = lifetimes.compress_lifetimes_days(lifetimes_map, compress_days) # Trim lifetimes based on latest price dates. for base_quote in lifetimes_map: intervals = lifetimes_map[base_quote] result = prices.get_latest_price(price_map, base_quote) if result is None or result[0] is None: lifetimes_map[base_quote] = lifetimes.trim_intervals(intervals, None, date_last) else: latest_price_date = result[0] date_first = latest_price_date + datetime.timedelta(days=1) if date_first < date_last: lifetimes_map[base_quote] = lifetimes.trim_intervals( intervals, date_first, date_last ) else: # We don't need to update if we're already up to date. lifetimes_map[base_quote] = [] # Remove currency pairs we can't fetch any prices for. if not default_source: keys = list(lifetimes_map.keys()) for key in keys: if not currency_map.get(key, None): del lifetimes_map[key] # Create price jobs based on fetch rate if update_rate == "daily": required_prices = lifetimes.required_daily_prices( lifetimes_map, date_last, weekdays_only=False ) elif update_rate == "weekday": required_prices = lifetimes.required_daily_prices( lifetimes_map, date_last, weekdays_only=True ) elif update_rate == "weekly": required_prices = lifetimes.required_weekly_prices(lifetimes_map, date_last) else: raise ValueError("Invalid Update Rate") jobs = [] # Build up the list of jobs to fetch prices for. for key in required_prices: date, base, quote = key psources = currency_map.get((base, quote), None) if not psources: psources = [PriceSource(default_source, base, False)] jobs.append(DatedPrice(base, quote, date, psources)) return sorted(jobs) def now(): "Indirection in order to be able to mock it out in the tests." return datetime.datetime.now(datetime.timezone.utc) def fetch_cached_price(source, symbol, date): """Call Source to fetch a price, but look and/or update the cache first. This function entirely deals with caching and correct expiration. It keeps old prices if they were fetched in the past, and it quickly expires intra-day prices if they are fetched on the same day. Args: source: A Python module object. symbol: A string, the ticker to fetch. date: A datetime.date instance, None if we're to fetch the latest date. Returns: A SourcePrice instance. """ # Compute a suitable timestamp from the date, if specified. if date is not None: # We query as for 4pm for the given date of the current timezone, if # specified. query_time = datetime.time(16, 0, 0) time_local = datetime.datetime.combine(date, query_time, tzinfo=tz.tzlocal()) time = time_local.astimezone(tz.tzutc()) else: time = None if _CACHE is None: # The cache is disabled; just call and return. result = ( source.get_latest_price(symbol) if time is None else source.get_historical_price(symbol, time) ) else: # The cache is enabled and we have to compute the current/latest price. # Try to fetch from the cache but miss if the price is too old. md5 = hashlib.md5() md5.update(str((type(source).__module__, symbol, date)).encode("utf-8")) key = md5.hexdigest() timestamp_now = int(now().timestamp()) try: timestamp_created, result_naive = _CACHE[key] # Convert naive timezone to UTC, which is what the cache is always # assumed to store. (The reason for this is that timezones from # aware datetime objects cannot be serialized properly due to bug.) if result_naive.time is not None: result = result_naive._replace( time=result_naive.time.replace(tzinfo=tz.tzutc()) ) else: result = result_naive if (timestamp_now - timestamp_created) > _CACHE.expiration.total_seconds(): raise KeyError except KeyError: logging.info("Fetching: %s (time: %s)", symbol, time) try: result = ( source.get_latest_price(symbol) if time is None else source.get_historical_price(symbol, time) ) except ValueError as exc: logging.error("Error fetching %s: %s", symbol, exc) result = None # Make sure the timezone is UTC and make naive before serialization. if result and result.time is not None: time_utc = result.time.astimezone(tz.tzutc()) time_naive = time_utc.replace(tzinfo=None) result_naive = result._replace(time=time_naive) else: result_naive = result if result_naive is not None: _CACHE[key] = (timestamp_now, result_naive) return result def setup_cache(cache_filename: Optional[str], clear_cache: bool): """Setup the results cache. Args: cache_filename: A string or None, the base filename for the cache. An extension may be added to the filename and more than one file may be created. clear_cache: A boolean, if true, delete the cache before beginning. """ if not cache_filename: return logging.info('Using price cache at "%s" (with indefinite expiration)', cache_filename) flag = "c" if clear_cache and cache_filename: logging.info("Clearing cache %s*", cache_filename) flag = "n" global _CACHE _CACHE = shelve.open(cache_filename, flag=flag) # type: ignore _CACHE.expiration = DEFAULT_EXPIRATION # type: ignore def reset_cache(): """Reset the cache to its uninitialized state.""" global _CACHE if _CACHE is not None: _CACHE.close() _CACHE = None def fetch_price(dprice: DatedPrice, swap_inverted: bool = False) -> Optional[data.Price]: """Fetch a price for the DatedPrice job. Args: dprice: A DatedPrice instances. swap_inverted: A boolean, true if we should invert currencies instead of rate for an inverted price source. Returns: A Price entry corresponding to the output of the jobs processed. """ for psource in dprice.sources: try: source = psource.module.Source() except AttributeError: continue srcprice = fetch_cached_price(source, psource.symbol, dprice.date) if srcprice is not None: break else: if dprice.sources: logging.error("Could not fetch for job: %s", dprice) return None base = dprice.base quote = dprice.quote or srcprice.quote_currency price = srcprice.price # Invert the rate if requested. if psource.invert: if swap_inverted: base, quote = quote, base else: price = ONE / price assert base is not None fileloc = data.new_metadata("<{}>".format(type(psource.module).__name__), 0) # The datetime instance is required to be aware. We always convert to the # user's timezone before extracting the date. This means that if the market # returns a timestamp for a particular date, once we convert to the user's # timezone the returned date may be different by a day. The intent is that # whatever we print is assumed coherent with the user's timezone. See # discussion at # https://groups.google.com/d/msg/beancount/9j1E_HLEMBQ/fYRuCQK_BwAJ srctime = srcprice.time if srctime.tzinfo is None: raise ValueError("Time returned by the price source is not timezone aware.") date = srctime.astimezone(tz.tzlocal()).date() return data.Price(fileloc, date, base, amount.Amount(price, quote or UNKNOWN_CURRENCY)) def filter_redundant_prices( price_entries: List[data.Price], existing_entries: List[data.Price], diffs: bool = False ) -> Tuple[List[data.Price], List[data.Price]]: """Filter out new entries that are redundant from an existing set. If the price differs, we override it with the new entry only on demand. This is because this would create conflict with existing price entries when parsing, if the new entries are simply inserted into the input. Args: price_entries: A list of newly created, proposed to be added Price directives. existing_entries: A list of existing entries we are proposing to add to. diffs: A boolean, true if we should output differing price entries at the same date. Returns: A filtered list of remaining entries, and a list of ignored entries. """ # Note: We have to be careful with the dates, because requesting the latest # price for a date may yield the price at a previous date. Clobber needs to # take this into account. See {1cfa25e37fc1}. existing_prices = { (entry.date, entry.currency): entry for entry in existing_entries if isinstance(entry, data.Price) } filtered_prices: List[data.Price] = [] ignored_prices: List[data.Price] = [] for entry in price_entries: key = (entry.date, entry.currency) if key in existing_prices: if diffs: existing_entry = existing_prices[key] if existing_entry.amount == entry.amount: output = ignored_prices else: output = ignored_prices else: output = filtered_prices output.append(entry) return filtered_prices, ignored_prices def process_args() -> Tuple[ argparse.Namespace, List[DatedPrice], data.Directives, Optional[Any], ]: """Process the arguments. This also initializes the logging module. Returns: A tuple of: args: The argparse receiver of command-line arguments. jobs: A list of DatedPrice job objects. entries: A list of all the parsed entries. dcontext: A context used to determine decimal precision when printing. """ parser = argparse.ArgumentParser(description=beanprice.__doc__.splitlines()[0]) # Input sources or filenames. parser.add_argument( "sources", nargs="+", help=( 'A list of filenames (or source "module/symbol", if -e is ' "specified) from which to create a list of jobs." ), ) parser.add_argument( "-e", "--expressions", "--expression", action="store_true", help=('Interpret the arguments as "module/symbol" source strings.'), ) # Regular options. parser.add_argument( "-v", "--verbose", action="count", help=("Print out progress log. Specify twice for debugging info."), ) parser.add_argument( "-d", "--date", action="store", type=date_utils.parse_date_liberally, help=("Specify the date for which to fetch the prices."), ) parser.add_argument( "--update", action="store_true", help=( "Fetch prices from most recent price for each source " "up to present day or specified --date. See also " "--update-rate, --update-compress options." ), ) parser.add_argument( "--update-rate", choices=["daily", "weekday", "weekly"], default="weekday", help=( "Specify how often dates are fetched. Options are daily, weekday, or weekly " "(fridays)" ), ) parser.add_argument( "--update-compress", action="store", type=int, default=0, help=( "Specify the number of inactive days to ignore. This option ignored if " "--inactive used." ), ) parser.add_argument( "-i", "--inactive", action="store_true", help=( "Select all commodities from input files, not just the ones active on the date" ), ) parser.add_argument( "-u", "--undeclared", action="store_true", help=( "Include commodities viewed in the file even without a " "corresponding Commodity directive, from this default source. " "The currency name itself is used as the lookup symbol in this default source." ), ) parser.add_argument( "-c", "--clobber", action="store_true", help=( "Do not skip prices which are already present in input files; " "fetch them anyway." ), ) parser.add_argument( "-a", "--all", action="store_true", help=("A shorthand for --inactive, --undeclared, --clobber."), ) parser.add_argument( "-s", "--swap-inverted", action="store_true", help=( "For inverted sources, swap currencies instead of inverting the rate. " "For example, if fetching the rate for CAD from 'USD:google/^CURRENCY:USDCAD' " 'results in 1.25, by default we would output "price CAD 0.8000 USD". ' 'Using this option we would instead output " price USD 1.2500 CAD".' ), ) parser.add_argument( "-w", "--workers", action="store", type=int, default=1, help=("Specify the number of concurrent fetchers."), ) parser.add_argument( "-n", "--dry-run", action="store_true", help=( "Don't actually fetch the prices, just print the list of the ones " "to be fetched." ), ) # Caching options. cache_group = parser.add_argument_group("cache") cache_filename = path.join( tempfile.gettempdir(), "{}.cache".format(path.basename(sys.argv[0])) ) cache_group.add_argument( "--cache", dest="cache_filename", action="store", default=cache_filename, help="The base filename for the underlying price cache " "database. An extension may be added to the filename and " "more than one file may be created.", ) cache_group.add_argument( "--no-cache", dest="cache_filename", action="store_const", const=None, help="Disable the price cache.", ) cache_group.add_argument( "--clear-cache", action="store_true", help="Clear the cache prior to startup." ) args = parser.parse_args() verbose_levels = { None: logging.WARN, 0: logging.WARN, 1: logging.INFO, 2: logging.DEBUG, } logging.basicConfig( level=verbose_levels[args.verbose], format="%(levelname)-8s: %(message)s" ) if args.undeclared: args.undeclared = DEFAULT_SOURCE if args.all: args.inactive = args.clobber = True args.undeclared = DEFAULT_SOURCE # Setup for processing. setup_cache(args.cache_filename, args.clear_cache) # Get the list of DatedPrice jobs to get from the arguments. dates = [args.date or None] logging.info("Processing at date: %s", args.date or datetime.date.today()) jobs = [] all_entries = [] dcontext = None if args.expressions: # Interpret the arguments as price sources. for source_str in args.sources: psources: List[PriceSource] = [] try: psource_map = parse_source_map(source_str) except ValueError: extra = "; did you provide a filename?" if path.exists(source_str) else "" msg = ( 'Invalid source "{{}}"{}. '.format(extra) + 'Supported format is "CCY:module/SYMBOL"' ) parser.error(msg.format(source_str)) else: for currency, psources in psource_map.items(): for date in dates: jobs.append( DatedPrice(psources[0].symbol, currency, date, psources) ) elif args.update: # Use Beancount input filename sources to create # prices jobs up to present time. for filename in args.sources: if not path.exists(filename) or not path.isfile(filename): parser.error( 'File does not exist: "{}"; ' "did you mean to use -e?".format(filename) ) continue logging.info('Loading "%s"', filename) entries, errors, options_map = loader.load_file(filename, log_errors=sys.stderr) if dcontext is None: dcontext = options_map["dcontext"] if args.date is None: latest_date = datetime.date.today() else: latest_date = args.date jobs.extend( get_price_jobs_up_to_date( entries, latest_date, args.inactive, args.undeclared, args.update_rate, args.update_compress, ) ) all_entries.extend(entries) else: # Interpret the arguments as Beancount input filenames. for filename in args.sources: if not path.exists(filename) or not path.isfile(filename): parser.error( 'File does not exist: "{}"; ' "did you mean to use -e?".format(filename) ) continue logging.info('Loading "%s"', filename) entries, errors, options_map = loader.load_file(filename, log_errors=sys.stderr) if dcontext is None: dcontext = options_map["dcontext"] for date in dates: jobs.extend( get_price_jobs_at_date(entries, date, args.inactive, args.undeclared) ) all_entries.extend(entries) return args, jobs, data.sorted(all_entries), dcontext def main(): args, jobs, entries, dcontext = process_args() # If we're just being asked to list the jobs, do this here. if args.dry_run: for dprice in jobs: print(format_dated_price_str(dprice)) return # Fetch all the required prices, processing all the jobs. executor = futures.ThreadPoolExecutor(max_workers=args.workers) price_entries = filter( None, executor.map( functools.partial(fetch_price, swap_inverted=args.swap_inverted), jobs ), ) # Sort them by currency, regardless of date (the dates should be close # anyhow, and we tend to put them in chunks in the input files anyhow). price_entries = sorted(price_entries, key=lambda e: e.currency) if args.update: # Sort additionally by date, to have an output consistent # with single date bean-price output. price_entries = sorted(price_entries, key=lambda e: e.date) # Avoid clobber, remove redundant entries. if not args.clobber: price_entries, ignored_entries = filter_redundant_prices(price_entries, entries) for entry in ignored_entries: logging.info("Ignored to avoid clobber: %s %s", entry.date, entry.currency) # Print out the entries. printer.print_entries(price_entries, dcontext=dcontext) beanprice-2.0.0/beanprice/price_test.py000066400000000000000000000601501475134044500201150ustar00rootroot00000000000000"""Tests for main driver for price fetching.""" __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime import logging import shutil import sys import tempfile import types import unittest from os import path from unittest import mock from decimal import Decimal from dateutil import tz from beancount.utils import test_utils from beancount.parser import cmptest from beancount import loader from beanprice.source import SourcePrice from beanprice import price from beanprice.sources import yahoo PS = price.PriceSource def run_with_args(function, args, runner_file=None): """Run the given function with sys.argv set to argv. The first argument is automatically inferred to be where the function object was defined. sys.argv is restored after the function is called. Args: function: A function object to call with no arguments. argv: A list of arguments, excluding the script name, to be temporarily set on sys.argv. runner_file: An optional name of the top-level file being run. Returns: The return value of the function run. """ saved_argv = sys.argv saved_handlers = logging.root.handlers try: if runner_file is None: module = sys.modules[function.__module__] runner_file = module.__file__ sys.argv = [runner_file] + args logging.root.handlers = [] return function() finally: sys.argv = saved_argv logging.root.handlers = saved_handlers class TestCache(unittest.TestCase): def test_fetch_cached_price__disabled(self): # Latest. with mock.patch("beanprice.price._CACHE", None): self.assertIsNone(price._CACHE) source = mock.MagicMock() price.fetch_cached_price(source, "HOOL", None) self.assertTrue(source.get_latest_price.called) # Historical. with mock.patch("beanprice.price._CACHE", None): self.assertIsNone(price._CACHE) source = mock.MagicMock() price.fetch_cached_price(source, "HOOL", datetime.date.today()) self.assertTrue(source.get_historical_price.called) def test_fetch_cached_price__latest(self): tmpdir = tempfile.mkdtemp() tmpfile = path.join(tmpdir, "prices.cache") try: price.setup_cache(tmpfile, False) srcprice = SourcePrice( Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" ) source = mock.MagicMock() source.get_latest_price.return_value = srcprice source.__file__ = "" # Cache miss. result = price.fetch_cached_price(source, "HOOL", None) self.assertTrue(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) source.get_latest_price.reset_mock() # Cache hit. result = price.fetch_cached_price(source, "HOOL", None) self.assertFalse(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) srcprice2 = SourcePrice( Decimal("1.894"), datetime.datetime.now(tz.tzutc()), "USD" ) source.get_latest_price.reset_mock() source.get_latest_price.return_value = srcprice2 # Cache expired. time_beyond = datetime.datetime.now() + price._CACHE.expiration * 2 with mock.patch("beanprice.price.now", return_value=time_beyond): result = price.fetch_cached_price(source, "HOOL", None) self.assertTrue(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice2, result) finally: price.reset_cache() if path.exists(tmpdir): shutil.rmtree(tmpdir) def test_fetch_cached_price__clear_cache(self): tmpdir = tempfile.mkdtemp() tmpfile = path.join(tmpdir, "prices.cache") try: price.setup_cache(tmpfile, False) srcprice = SourcePrice( Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" ) source = mock.MagicMock() source.get_latest_price.return_value = srcprice source.__file__ = "" # Cache miss. result = price.fetch_cached_price(source, "HOOL", None) self.assertTrue(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) source.get_latest_price.reset_mock() # Cache hit. result = price.fetch_cached_price(source, "HOOL", None) self.assertFalse(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) srcprice2 = SourcePrice( Decimal("1.894"), datetime.datetime.now(tz.tzutc()), "USD" ) source.get_latest_price.reset_mock() source.get_latest_price.return_value = srcprice2 # Open cache again, but clear it. price.reset_cache() price.setup_cache(tmpfile, True) # Cache cleared. result = price.fetch_cached_price(source, "HOOL", None) self.assertTrue(source.get_latest_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice2, result) finally: price.reset_cache() if path.exists(tmpdir): shutil.rmtree(tmpdir) def test_fetch_cached_price__historical(self): tmpdir = tempfile.mkdtemp() tmpfile = path.join(tmpdir, "prices.cache") try: price.setup_cache(tmpfile, False) srcprice = SourcePrice( Decimal("1.723"), datetime.datetime.now(tz.tzutc()), "USD" ) source = mock.MagicMock() source.get_historical_price.return_value = srcprice source.__file__ = "" # Cache miss. day = datetime.date(2006, 1, 2) result = price.fetch_cached_price(source, "HOOL", day) self.assertTrue(source.get_historical_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) source.get_historical_price.reset_mock() # Cache hit. result = price.fetch_cached_price(source, "HOOL", day) self.assertFalse(source.get_historical_price.called) self.assertEqual(1, len(price._CACHE)) self.assertEqual(srcprice, result) finally: price.reset_cache() if path.exists(tmpdir): shutil.rmtree(tmpdir) class TestProcessArguments(unittest.TestCase): def test_filename_not_exists(self): with test_utils.capture("stderr"): with self.assertRaises(SystemExit): run_with_args(price.process_args, ["--no-cache", "/some/file.beancount"]) @test_utils.docfile def test_explicit_file__badcontents(self, filename): """ 2015-01-01 open Assets:Invest 2015-01-01 open USD ;; Error """ with test_utils.capture("stderr"): args, jobs, _, __ = run_with_args(price.process_args, ["--no-cache", filename]) self.assertEqual([], jobs) def test_filename_exists(self): with tempfile.NamedTemporaryFile("w") as tmpfile: with test_utils.capture("stderr"): args, jobs, _, __ = run_with_args( price.process_args, ["--no-cache", tmpfile.name] ) self.assertEqual([], jobs) # Empty file. def test_expressions(self): with test_utils.capture("stderr"): args, jobs, _, __ = run_with_args( price.process_args, ["--no-cache", "-e", "USD:yahoo/AAPL"] ) self.assertEqual( [ price.DatedPrice( "AAPL", "USD", None, [price.PriceSource(yahoo, "AAPL", False)] ) ], jobs, ) class TestClobber(cmptest.TestCase): @loader.load_doc() def setUp(self, entries, _, __): """ ;; Existing file. 2015-01-05 price HDV 75.56 USD 2015-01-23 price HDV 77.34 USD 2015-02-06 price HDV 77.16 USD 2015-02-12 price HDV 78.17 USD 2015-05-01 price HDV 77.48 USD 2015-06-02 price HDV 76.33 USD 2015-06-29 price HDV 73.74 USD 2015-07-06 price HDV 73.79 USD 2015-08-11 price HDV 74.19 USD 2015-09-04 price HDV 68.98 USD """ self.entries = entries # New entries. self.price_entries, _, __ = loader.load_string( """ 2015-01-27 price HDV 76.83 USD 2015-02-06 price HDV 77.16 USD 2015-02-19 price HDV 77.5 USD 2015-06-02 price HDV 76.33 USD 2015-06-19 price HDV 76 USD 2015-07-06 price HDV 73.79 USD 2015-07-31 price HDV 74.64 USD 2015-08-11 price HDV 74.20 USD ;; Different """, dedent=True, ) def test_clobber_nodiffs(self): new_price_entries, _ = price.filter_redundant_prices( self.price_entries, self.entries, diffs=False ) self.assertEqualEntries( """ 2015-01-27 price HDV 76.83 USD 2015-02-19 price HDV 77.5 USD 2015-06-19 price HDV 76 USD 2015-07-31 price HDV 74.64 USD """, new_price_entries, ) def test_clobber_diffs(self): new_price_entries, _ = price.filter_redundant_prices( self.price_entries, self.entries, diffs=True ) self.assertEqualEntries( """ 2015-01-27 price HDV 76.83 USD 2015-02-19 price HDV 77.5 USD 2015-06-19 price HDV 76 USD 2015-07-31 price HDV 74.64 USD 2015-08-11 price HDV 74.20 USD ;; Different """, new_price_entries, ) class TestTimezone(unittest.TestCase): @mock.patch.object(price, "fetch_cached_price") def test_fetch_price__naive_time_no_timeozne(self, fetch_cached): fetch_cached.return_value = SourcePrice( Decimal("125.00"), datetime.datetime(2015, 11, 22, 16, 0, 0), "JPY" ) dprice = price.DatedPrice("JPY", "USD", datetime.date(2015, 11, 22), None) with self.assertRaises(ValueError): price.fetch_price( dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", False)]), False ) class TestInverted(unittest.TestCase): def setUp(self): fetch_cached = mock.patch("beanprice.price.fetch_cached_price").start() fetch_cached.return_value = SourcePrice( Decimal("125.00"), datetime.datetime(2015, 11, 22, 16, 0, 0, tzinfo=tz.tzlocal()), "JPY", ) self.dprice = price.DatedPrice("JPY", "USD", datetime.date(2015, 11, 22), None) self.addCleanup(mock.patch.stopall) def test_fetch_price__normal(self): entry = price.fetch_price( self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", False)]), False ) self.assertEqual(("JPY", "USD"), (entry.currency, entry.amount.currency)) self.assertEqual(Decimal("125.00"), entry.amount.number) def test_fetch_price__inverted(self): entry = price.fetch_price( self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", True)]), False ) self.assertEqual(("JPY", "USD"), (entry.currency, entry.amount.currency)) self.assertEqual(Decimal("0.008"), entry.amount.number) def test_fetch_price__swapped(self): entry = price.fetch_price( self.dprice._replace(sources=[price.PriceSource(yahoo, "USDJPY", True)]), True ) self.assertEqual(("USD", "JPY"), (entry.currency, entry.amount.currency)) self.assertEqual(Decimal("125.00"), entry.amount.number) class TestImportSource(unittest.TestCase): def test_import_source_valid(self): for name in "oanda", "yahoo": module = price.import_source(name) self.assertIsInstance(module, types.ModuleType) module = price.import_source("beanprice.sources.yahoo") self.assertIsInstance(module, types.ModuleType) def test_import_source_invalid(self): with self.assertRaises(ImportError): price.import_source("non.existing.module") class TestParseSource(unittest.TestCase): def test_source_invalid(self): with self.assertRaises(ValueError): price.parse_single_source("AAPL") with self.assertRaises(ValueError): price.parse_single_source("***//--") # The module gets imported at this stage. with self.assertRaises(ImportError): price.parse_single_source("invalid.module.name/NASDAQ:AAPL") def test_source_valid(self): psource = price.parse_single_source("yahoo/CNYUSD=X") self.assertEqual(PS(yahoo, "CNYUSD=X", False), psource) # Make sure that an invalid name at the tail doesn't succeed. with self.assertRaises(ValueError): psource = price.parse_single_source("yahoo/CNYUSD&X") psource = price.parse_single_source("beanprice.sources.yahoo/AAPL") self.assertEqual(PS(yahoo, "AAPL", False), psource) class TestParseSourceMap(unittest.TestCase): def _clean_source_map(self, smap): return { currency: [PS(s[0].__name__, s[1], s[2]) for s in sources] for currency, sources in smap.items() } def test_source_map_invalid(self): for expr in "USD", "something else", "USD:NASDAQ:AAPL": with self.assertRaises(ValueError): price.parse_source_map(expr) def test_source_map_onecur_single(self): smap = price.parse_source_map("USD:yahoo/AAPL") self.assertEqual( {"USD": [PS("beanprice.sources.yahoo", "AAPL", False)]}, self._clean_source_map(smap), ) def test_source_map_onecur_multiple(self): smap = price.parse_source_map("USD:oanda/USDCAD,yahoo/CAD=X") self.assertEqual( { "USD": [ PS("beanprice.sources.oanda", "USDCAD", False), PS("beanprice.sources.yahoo", "CAD=X", False), ] }, self._clean_source_map(smap), ) def test_source_map_manycur_single(self): smap = price.parse_source_map("USD:yahoo/USDCAD CAD:yahoo/CAD=X") self.assertEqual( { "USD": [PS("beanprice.sources.yahoo", "USDCAD", False)], "CAD": [PS("beanprice.sources.yahoo", "CAD=X", False)], }, self._clean_source_map(smap), ) def test_source_map_manycur_multiple(self): smap = price.parse_source_map("USD:yahoo/GBPUSD,oanda/GBPUSD CAD:yahoo/GBPCAD") self.assertEqual( { "USD": [ PS("beanprice.sources.yahoo", "GBPUSD", False), PS("beanprice.sources.oanda", "GBPUSD", False), ], "CAD": [PS("beanprice.sources.yahoo", "GBPCAD", False)], }, self._clean_source_map(smap), ) def test_source_map_inverse(self): smap = price.parse_source_map("USD:yahoo/^GBPUSD") self.assertEqual( {"USD": [PS("beanprice.sources.yahoo", "GBPUSD", True)]}, self._clean_source_map(smap), ) class TestFilters(unittest.TestCase): @loader.load_doc() def test_get_price_jobs__date(self, entries, _, __): """ 2000-01-10 open Assets:US:Invest:QQQ 2000-01-10 open Assets:US:Invest:VEA 2000-01-10 open Assets:US:Invest:Margin 2014-01-01 commodity QQQ price: "USD:yahoo/NASDAQ:QQQ" 2014-01-01 commodity VEA price: "USD:yahoo/NASDAQ:VEA" 2014-02-06 * Assets:US:Invest:QQQ 100 QQQ {86.23 USD} Assets:US:Invest:VEA 200 VEA {43.22 USD} Assets:US:Invest:Margin 2014-08-07 * Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD Assets:US:Invest:Margin 2015-01-15 * Assets:US:Invest:QQQ 10 QQQ {92.32 USD} Assets:US:Invest:VEA -200 VEA {43.22 USD} @ 41.01 USD Assets:US:Invest:Margin """ jobs = price.get_price_jobs_at_date(entries, datetime.date(2014, 1, 1), False, None) self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) jobs = price.get_price_jobs_at_date(entries, datetime.date(2014, 6, 1), False, None) self.assertEqual( {("QQQ", "USD"), ("VEA", "USD")}, {(job.base, job.quote) for job in jobs} ) jobs = price.get_price_jobs_at_date( entries, datetime.date(2014, 10, 1), False, None ) self.assertEqual({("VEA", "USD")}, {(job.base, job.quote) for job in jobs}) jobs = price.get_price_jobs_at_date(entries, None, False, None) self.assertEqual({("QQQ", "USD")}, {(job.base, job.quote) for job in jobs}) @loader.load_doc() def test_get_price_jobs__inactive(self, entries, _, __): """ 2000-01-10 open Assets:US:Invest:QQQ 2000-01-10 open Assets:US:Invest:VEA 2000-01-10 open Assets:US:Invest:Margin 2014-01-01 commodity QQQ price: "USD:yahoo/NASDAQ:QQQ" 2014-01-01 commodity VEA price: "USD:yahoo/NASDAQ:VEA" 2014-02-06 * Assets:US:Invest:QQQ 100 QQQ {86.23 USD} Assets:US:Invest:VEA 200 VEA {43.22 USD} Assets:US:Invest:Margin 2014-08-07 * Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD Assets:US:Invest:Margin """ jobs = price.get_price_jobs_at_date(entries, None, False, None) self.assertEqual({("VEA", "USD")}, {(job.base, job.quote) for job in jobs}) jobs = price.get_price_jobs_at_date(entries, None, True, None) self.assertEqual( {("VEA", "USD"), ("QQQ", "USD")}, {(job.base, job.quote) for job in jobs} ) @loader.load_doc() def test_get_price_jobs__undeclared(self, entries, _, __): """ 2000-01-10 open Assets:US:Invest:QQQ 2000-01-10 open Assets:US:Invest:VEA 2000-01-10 open Assets:US:Invest:Margin 2014-01-01 commodity QQQ price: "USD:yahoo/NASDAQ:QQQ" 2014-02-06 * Assets:US:Invest:QQQ 100 QQQ {86.23 USD} Assets:US:Invest:VEA 200 VEA {43.22 USD} Assets:US:Invest:Margin """ jobs = price.get_price_jobs_at_date(entries, None, False, None) self.assertEqual({("QQQ", "USD")}, {(job.base, job.quote) for job in jobs}) jobs = price.get_price_jobs_at_date(entries, None, False, "yahoo") self.assertEqual( {("QQQ", "USD"), ("VEA", "USD")}, {(job.base, job.quote) for job in jobs} ) @loader.load_doc() def test_get_price_jobs__default_source(self, entries, _, __): """ 2000-01-10 open Assets:US:Invest:QQQ 2000-01-10 open Assets:US:Invest:Margin 2014-01-01 commodity QQQ price: "NASDAQ:QQQ" 2014-02-06 * Assets:US:Invest:QQQ 100 QQQ {86.23 USD} Assets:US:Invest:Margin """ jobs = price.get_price_jobs_at_date(entries, None, False, "yahoo") self.assertEqual(1, len(jobs[0].sources)) self.assertIsInstance(jobs[0].sources[0], price.PriceSource) @loader.load_doc() def test_get_price_jobs__currencies_not_at_cost(self, entries, _, __): """ 2000-01-10 open Assets:US:BofA:Checking 2000-01-10 open Assets:US:BofA:CHF 2014-01-01 commodity USD 2014-01-01 commodity CHF price: "USD:yahoo/CHFUSD=X" 2021-01-04 * Assets:US:BofA:Checking 100 USD Assets:US:BofA:CHF -110 CHF @@ 100 USD """ # TODO: Shouldn't we actually return (CHF, USD) here? jobs = price.get_price_jobs_at_date(entries, datetime.date(2021, 1, 4), False, None) self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) jobs = price.get_price_jobs_at_date(entries, datetime.date(2021, 1, 6), False, None) self.assertEqual({("CHF", "USD")}, {(job.base, job.quote) for job in jobs}) # TODO: Shouldn't we return (CHF, USD) here, as above? jobs = price.get_price_jobs_up_to_date( entries, datetime.date(2021, 1, 6), False, None ) self.assertEqual(set(), {(job.base, job.quote) for job in jobs}) @loader.load_doc() def test_get_price_jobs_up_to_date(self, entries, _, __): """ 2000-01-10 open Assets:US:Invest:QQQ 2000-01-10 open Assets:US:Invest:VEA 2000-01-10 open Assets:US:Invest:Margin 2021-01-01 commodity QQQ price: "USD:yahoo/NASDAQ:QQQ" 2021-01-01 commodity VEA price: "USD:yahoo/NASDAQ:VEA" 2021-01-04 * Assets:US:Invest:QQQ 100 QQQ {86.23 USD} Assets:US:Invest:VEA 200 VEA {43.22 USD} Assets:US:Invest:Margin 2021-01-05 * Assets:US:Invest:QQQ -100 QQQ {86.23 USD} @ 91.23 USD Assets:US:Invest:Margin 2021-01-07 * Assets:US:Invest:QQQ 10 QQQ {92.32 USD} Assets:US:Invest:VEA -200 VEA {43.22 USD} @ 41.01 USD Assets:US:Invest:Margin """ jobs = price.get_price_jobs_up_to_date(entries, datetime.date(2021, 1, 8)) self.assertEqual( { ("QQQ", "USD", datetime.date(2021, 1, 4)), ("QQQ", "USD", datetime.date(2021, 1, 5)), ("QQQ", "USD", datetime.date(2021, 1, 7)), ("VEA", "USD", datetime.date(2021, 1, 4)), ("VEA", "USD", datetime.date(2021, 1, 5)), ("VEA", "USD", datetime.date(2021, 1, 6)), ("VEA", "USD", datetime.date(2021, 1, 7)), }, {(job.base, job.quote, job.date) for job in jobs}, ) class TestFromFile(unittest.TestCase): @loader.load_doc() def setUp(self, entries, _, __): """ 2000-01-10 open Assets:US:Investments:QQQ 2000-01-10 open Assets:CA:Investments:XSP 2000-01-10 open Assets:Cash 2000-01-10 open Assets:External 2000-01-10 open Expenses:Foreign 2010-01-01 commodity USD 2010-01-01 commodity QQQ name: "PowerShares QQQ Trust, Series 1 (ETF)" price: "USD:yahoo/NASDAQ:QQQ" 2010-01-01 commodity XSP name: "iShares S&P 500 Index Fund (CAD Hedged)" quote: CAD 2010-01-01 commodity AMTKPTS quote: USD price: "" """ self.entries = entries def test_find_currencies_declared(self): currencies = price.find_currencies_declared(self.entries, None) currencies2 = [(base, quote) for base, quote, _ in currencies] self.assertEqual([("QQQ", "USD")], currencies2) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/source.py000066400000000000000000000107301475134044500172530ustar00rootroot00000000000000"""Interface definition for all price sources. This module describes the contract to be fulfilled by all implementations of price sources. TODO(blais): It would be an improvement if the interfaces here return an indication of why fetching failed and leave the responsibility to the caller to decide whether to share this with the user or to ignore and continue with other sources. """ __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime from decimal import Decimal from typing import List, Optional, NamedTuple # A record that contains data for a price fetched from a source. # # A triple of # price: A Decimal instance, the price or rate. # time: A datetime.time instance at which that price or rate was available. # Note that this instance is REQUIRED to be timezone aware, as this is # used to compute a corresponding date in the user's timezone. # quote-currency: A string, the quote currency of the given price, if # available. SourcePrice = NamedTuple( "SourcePrice", [ ("price", Decimal), ("time", Optional[datetime.datetime]), ("quote_currency", Optional[str]), ], ) class Source: """Interface to be implemented by all price sources. Notes about arguments below: `ticker` arguments: A string, the ticker to be fetched by the source. This ticker may include structure, such as the exchange code. Also note that this ticker is source-specified, and is not necessarily the same value as the commodity symbol used in the Beancount file. time arguments: A `datetime.datetime` instance. This is a timezone-aware `datetime` you can convert to any timezone. For past dates we query for a time that is equivalent to 4pm in the user's timezone. About return values: If the price could not be fetched, None is returned and another source should be consulted. There is never any guarantee that a price source will be able to fetch its value and failure to fetch is more frequent than one might assume in practice; client code must be able to handle this and try again with another price source until all sources are exhausted. Also, note in the case we were able to fetch, the price's returned time must be timezone-aware (not naive). """ def get_latest_price(self, ticker: str) -> Optional[SourcePrice]: """Fetch the current latest price. The date may differ. This routine attempts to fetch the most recent available price, and returns the actual date of the quoted price, which may differ from the date this call is made at. {1cfa25e37fc1} Args: ticker: A string, the ticker to be fetched by the source. Returns: A SourcePrice instance, or None if we failed to fetch. """ def get_historical_price( self, ticker: str, time: datetime.datetime ) -> Optional[SourcePrice]: """Return the lastest historical price found for the symbol at the given date. This could be the price of the close of the day, for instance. We assume that there is some single price representative of the day. Also note that if you're querying for a weekend or holiday (closed market) date, the price returned may have a date earlier than the one you requested (the latest available market price for that instrument is from a prior date). Args: ticker: A string, the ticker to be fetched by the source. time: The timestamp at which to query for the price. Returns: A SourcePrice instance, or None if we failed to fetch. """ def get_prices_series( self, ticker: str, time_begin: datetime.datetime, time_end: datetime.datetime ) -> Optional[List[SourcePrice]]: """Return the historical daily price series between two dates. Note that weekends don't have any prices, so there's no guarantee that this returns a contiguous series of prices for each day in the requested interval. Args: ticker: A string, the ticker to be fetched by the source. time_begin: The earliest timestamp whose prices to include. time_end: The latest timestamp whose prices to include. Returns: A list of SourcePrice instances, sorted by date/time, or None if we failed to fetch. An empty list signals success fetching but no data in the requested interval. """ beanprice-2.0.0/beanprice/sources/000077500000000000000000000000001475134044500170635ustar00rootroot00000000000000beanprice-2.0.0/beanprice/sources/BUILD000066400000000000000000000043461475134044500176540ustar00rootroot00000000000000package(default_visibility = ["//visibility:public"]) py_library( name = "iex", srcs = ["iex.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", ], ) py_test( name = "iex_test", srcs = ["iex_test.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", "//beancount/prices/sources:iex", "//beanprice:date_utils", ], ) py_library( name = "oanda", srcs = ["oanda.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", "//beancount/utils:net_utils", ], ) py_test( name = "oanda_test", srcs = ["oanda_test.py"], deps = [ "//beanprice:date_utils", "//beancount/core:number", "//beancount/prices:source", "//beancount/prices/sources:oanda", "//beancount/utils:net_utils", ], ) py_library( name = "quandl", srcs = ["quandl.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", ], ) py_test( name = "quandl_test", srcs = ["quandl_test.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", "//beancount/prices/sources:quandl", "//beanprice:date_utils", ], ) py_library( name = "yahoo", srcs = ["yahoo.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", ], ) py_test( name = "yahoo_test", srcs = ["yahoo_test.py"], deps = [ "//beancount/core:number", "//beancount/prices/sources:yahoo", "//beanprice:date_utils", ], ) py_library( name = "tsp", srcs = ["tsp.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", ], ) py_test( name = "tsp_test", srcs = ["tsp_test.py"], deps = [ "//beancount/core:number", "//beancount/prices/sources:tsp", ], ) py_library( name = "coincap", srcs = ["coincap.py"], deps = [ "//beancount/core:number", "//beancount/prices:source", ], ) py_test( name = "coincap_test", srcs = ["coincap_test.py"], deps = [ "//beancount/core:number", "//beancount/prices/sources:tsp", ], ) beanprice-2.0.0/beanprice/sources/__init__.py000066400000000000000000000003341475134044500211740ustar00rootroot00000000000000"""Implementation of various price extractors. This package is looked up by the driver script to figure out which extractor to use. """ __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" beanprice-2.0.0/beanprice/sources/alphavantage.py000066400000000000000000000065151475134044500220770ustar00rootroot00000000000000"""A source fetching prices and exchangerates from https://www.alphavantage.co. It requires a free api key which needs to be set in the environment variable "ALPHAVANTAGE_API_KEY" Valid tickers for prices are in the form "price:XXX:YYY", such as "price:IBM:USD" where XXX is the symbol and YYY is the expected quote currency in which the data is returned. The api currently does not support converting to a specific ccy and does unfortunately not return in which ccy the result is. Valid tickers for exchangerates are in the form "fx:XXX:YYY", such as "fx:USD:CHF". Here is the API documentation: https://www.alphavantage.co/documentation/ For example: https://www.alphavantage.co/query?function=GLOBAL_QUOTE&symbol=IBM&apikey=demo https://www.alphavantage.co/query?function=CURRENCY_EXCHANGE_RATE&from_currency=USD&to_currency=JPY&apikey=demo """ from decimal import Decimal import re from os import environ from time import sleep import requests from dateutil.tz import tz from dateutil.parser import parse from beanprice import source class AlphavantageApiError(ValueError): "An error from the Alphavantage API." def _parse_ticker(ticker): """Parse the base and quote currencies from the ticker. Args: ticker: A string, the symbol in kind-XXX-YYY format. Returns: A (kind, symbol, base) tuple. """ match = re.match(r"^(?Pprice|fx):(?P[^:]+):(?P\w+)$", ticker) if not match: raise ValueError('Invalid ticker. Use "price:SYMBOL:BASE" or "fx:CCY:BASE" format.') return match.groups() def _do_fetch(params): params["apikey"] = environ["ALPHAVANTAGE_API_KEY"] resp = requests.get(url="https://www.alphavantage.co/query", params=params) data = resp.json() # This is for dealing with the rate limit, sleep for 60 seconds and then retry if "Note" in data: sleep(60) resp = requests.get(url="https://www.alphavantage.co/query", params=params) data = resp.json() if resp.status_code != requests.codes.ok: raise AlphavantageApiError( "Invalid response ({}): {}".format(resp.status_code, resp.text) ) if "Error Message" in data: raise AlphavantageApiError("Invalid response: {}".format(data["Error Message"])) return data class Source(source.Source): def get_latest_price(self, ticker): kind, symbol, base = _parse_ticker(ticker) if kind == "price": params = { "function": "GLOBAL_QUOTE", "symbol": symbol, } data = _do_fetch(params) price_data = data["Global Quote"] price = Decimal(price_data["05. price"]) date = parse(price_data["07. latest trading day"]).replace(tzinfo=tz.tzutc()) else: params = { "function": "CURRENCY_EXCHANGE_RATE", "from_currency": symbol, "to_currency": base, } data = _do_fetch(params) price_data = data["Realtime Currency Exchange Rate"] price = Decimal(price_data["5. Exchange Rate"]) date = parse(price_data["6. Last Refreshed"]).replace( tzinfo=tz.gettz(price_data["7. Time Zone"]) ) return source.SourcePrice(price, date, base) def get_historical_price(self, ticker, time): return None beanprice-2.0.0/beanprice/sources/alphavantage_test.py000066400000000000000000000053361475134044500231360ustar00rootroot00000000000000import datetime import unittest from os import environ from decimal import Decimal from unittest import mock from dateutil import tz import requests from beanprice import source from beanprice.sources import alphavantage def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class AlphavantagePriceFetcher(unittest.TestCase): def setUp(self): environ["ALPHAVANTAGE_API_KEY"] = "foo" def tearDown(self): del environ["ALPHAVANTAGE_API_KEY"] def test_error_invalid_ticker(self): with self.assertRaises(ValueError): alphavantage.Source().get_latest_price("INVALID") def test_error_network(self): with response("Foobar", 404): with self.assertRaises(alphavantage.AlphavantageApiError): alphavantage.Source().get_latest_price("price:IBM:USD") def test_error_response(self): contents = {"Error Message": "Something wrong"} with response(contents): with self.assertRaises(alphavantage.AlphavantageApiError): alphavantage.Source().get_latest_price("price:IBM:USD") def test_valid_response_price(self): contents = { "Global Quote": { "05. price": "144.7400", "07. latest trading day": "2021-01-21", } } with response(contents): srcprice = alphavantage.Source().get_latest_price("price:FOO:USD") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("144.7400"), srcprice.price) self.assertEqual("USD", srcprice.quote_currency) self.assertEqual( datetime.datetime(2021, 1, 21, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time ) def test_valid_response_fx(self): contents = { "Realtime Currency Exchange Rate": { "5. Exchange Rate": "108.94000000", "6. Last Refreshed": "2021-02-21 20:32:25", "7. Time Zone": "UTC", } } with response(contents): srcprice = alphavantage.Source().get_latest_price("fx:USD:CHF") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("108.94000000"), srcprice.price) self.assertEqual("CHF", srcprice.quote_currency) self.assertEqual( datetime.datetime(2021, 2, 21, 20, 32, 25, tzinfo=tz.tzutc()), srcprice.time ) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/coinbase.py000066400000000000000000000031051475134044500212170ustar00rootroot00000000000000"""A source fetching cryptocurrency prices from Coinbase. Valid tickers are in the form "XXX-YYY", such as "BTC-USD". Here is the API documentation: https://developers.coinbase.com/api/v2 For example: https://api.coinbase.com/v2/prices/BTC-GBP/spot Timezone information: Input and output datetimes are specified via UTC timestamps. """ import datetime from decimal import Decimal import requests from dateutil.tz import tz from beanprice import source class CoinbaseError(ValueError): "An error from the Coinbase API." def fetch_quote(ticker, time=None): """Fetch a quote from Coinbase.""" url = "https://api.coinbase.com/v2/prices/{}/spot".format(ticker.lower()) options = {} if time is not None: options["date"] = time.astimezone(tz.tzutc()).date().isoformat() response = requests.get(url, options) if response.status_code != requests.codes.ok: raise CoinbaseError( "Invalid response ({}): {}".format(response.status_code, response.text) ) result = response.json() price = Decimal(result["data"]["amount"]) if time is None: time = datetime.datetime.now(tz.tzutc()) currency = result["data"]["currency"] return source.SourcePrice(price, time, currency) class Source(source.Source): "Coinbase API price extractor." def get_latest_price(self, ticker): """See contract in beanprice.source.Source.""" return fetch_quote(ticker) def get_historical_price(self, ticker, time): """See contract in beanprice.source.Source.""" return fetch_quote(ticker, time) beanprice-2.0.0/beanprice/sources/coinbase_test.py000066400000000000000000000035401475134044500222610ustar00rootroot00000000000000import datetime import unittest from decimal import Decimal from unittest import mock from dateutil import tz import requests from beanprice import source from beanprice.sources import coinbase def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class CoinbasePriceFetcher(unittest.TestCase): def test_error_network(self): with response(None, 404): with self.assertRaises(ValueError) as exc: coinbase.fetch_quote("AAPL") self.assertRegex(exc.message, "premium") def test_valid_response(self): contents = {"data": {"base": "BTC", "currency": "USD", "amount": "101.23456"}} with response(contents): srcprice = coinbase.Source().get_latest_price("BTC-GBP") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("101.23456"), srcprice.price) self.assertEqual("USD", srcprice.quote_currency) def test_historical_price(self): contents = {"data": {"base": "BTC", "currency": "USD", "amount": "101.23456"}} with response(contents): time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) srcprice = coinbase.Source().get_historical_price("BTC-GBP", time) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("101.23456"), srcprice.price) self.assertEqual("USD", srcprice.quote_currency) self.assertEqual( datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time ) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/coincap.py000066400000000000000000000104151475134044500210520ustar00rootroot00000000000000""" A source fetching cryptocurrency prices from Coincap. Tickers can be in two formats: Either using the coincap currency id (bitcoin, ethereum, etc.), or using a currency ticker (BTC, ETH, etc.). In the latter case, any ambiguity will be resolved using the coin ranking. Prices are denoted in USD. The documentation can be found here: https://docs.coincap.io/ """ from datetime import datetime, timezone, timedelta import math from decimal import Decimal from typing import List, Optional, Dict import requests from beanprice import source API_BASE_URL = "https://api.coincap.io/v2/" class CoincapError(ValueError): "An error from the Coincap importer." def get_asset_list() -> List[Dict[str, str]]: """ Get list of currencies supported by Coincap. Returned is a list with elements with many properties, including "id", representing the Coincap id, and "symbol", representing the ticker symbol. """ path = "assets/" url = API_BASE_URL + path response = requests.get(url) data = response.json()["data"] return data def get_currency_id(currency: str) -> Optional[str]: """ Find currency ID by its symbol. If results are ambiguous, select currency with the highest market cap """ # Array is already sorted based on market cap for coin in get_asset_list(): if coin["symbol"] == currency: return coin["id"] return None def resolve_currency_id(base_currency: str) -> str: """ Obtain the currency ID from the ticker, which can either already be a currency id (bitcoin), or a coin ticker (BTC). """ if base_currency.isupper(): # Try to find currency ID by its symbol base_currency_id = get_currency_id(base_currency) if not isinstance(base_currency_id, str): raise CoincapError( f"Could not find currency id with ticker '{base_currency}'" ) return base_currency_id else: return base_currency def get_latest_price(base_currency: str) -> source.SourcePrice: """ Get the latest available price for a given currency. """ path = "assets/" url = f"{API_BASE_URL}{path}{resolve_currency_id(base_currency)}" response = requests.get(url) data = response.json() time = datetime.fromtimestamp(data["timestamp"] / 1000.0).replace( tzinfo=timezone.utc ) price = Decimal(data["data"]["priceUsd"]) return source.SourcePrice(price, time, "USD") def get_price_series( base_currency_id: str, time_begin: datetime, time_end: datetime ) -> List[source.SourcePrice]: path = f"assets/{base_currency_id}/history" params = { "interval": "d1", "start": str(math.floor(time_begin.timestamp() * 1000.0)), "end": str(math.ceil(time_end.timestamp() * 1000.0)), } url = API_BASE_URL + path response = requests.get(url, params=params) return [ source.SourcePrice( Decimal(item["priceUsd"]), datetime.fromtimestamp(item["time"] / 1000.0).replace(tzinfo=timezone.utc), "USD", ) for item in response.json()["data"] ] class Source(source.Source): """A price source for the Coincap API v2. Supports only prices denoted in USD. There are two ways of expressing a ticker, either by their coincap id (bitcoin) or by their ticker (BTC), in which case the highest ranked coin will be picked.""" def get_latest_price(self, ticker) -> source.SourcePrice: return get_latest_price(ticker) def get_historical_price( self, ticker: str, time: datetime ) -> Optional[source.SourcePrice]: for datapoint in self.get_prices_series( ticker, time + timedelta(days=-1), time + timedelta(days=1), ): # TODO(blais): This is poorly thought out, the date may not match # that in the differing timezone. You probably want the last price # before the datapoint time. if datapoint.time is not None and datapoint.time.date() == time.date(): return datapoint return None def get_prices_series( self, ticker: str, time_begin: datetime, time_end: datetime ) -> List[source.SourcePrice]: return get_price_series(resolve_currency_id(ticker), time_begin, time_end) beanprice-2.0.0/beanprice/sources/coincap_test.py000066400000000000000000000114141475134044500221110ustar00rootroot00000000000000import datetime import unittest from decimal import Decimal from unittest import mock from dateutil import tz import requests from beanprice import source from beanprice.sources import coincap timezone = tz.gettz("Europe/Amsterdam") response_assets_bitcoin_historical = { "data": [ { "priceUsd": "32263.2648195597839546", "time": 1609804800000, "date": "2021-01-05T00:00:00.000Z", }, { "priceUsd": "34869.7692419204775049", "time": 1609891200000, "date": "2021-01-06T00:00:00.000Z", }, ], "timestamp": 1618220568799, } response_assets_bitcoin = { "data": { "id": "bitcoin", "rank": "1", "symbol": "BTC", "name": "Bitcoin", "supply": "18672456.0000000000000000", "maxSupply": "21000000.0000000000000000", "marketCapUsd": "1134320211245.9295410753733840", "volumeUsd24Hr": "16998481452.4370929843940509", "priceUsd": "60748.3135183678858890", "changePercent24Hr": "1.3457951950518293", "vwap24Hr": "59970.0332730340881967", "explorer": "https://blockchain.info/", }, "timestamp": 1618218375359, } response_bitcoin_history = { "data": [ { "priceUsd": "29232.6707650537687673", "time": 1609459200000, "date": "2021-01-01T00:00:00.000Z", }, { "priceUsd": "30688.0967118388768791", "time": 1609545600000, "date": "2021-01-02T00:00:00.000Z", }, { "priceUsd": "33373.7277104175704785", "time": 1609632000000, "date": "2021-01-03T00:00:00.000Z", }, { "priceUsd": "31832.6862288485383625", "time": 1609718400000, "date": "2021-01-04T00:00:00.000Z", }, { "priceUsd": "32263.2648195597839546", "time": 1609804800000, "date": "2021-01-05T00:00:00.000Z", }, { "priceUsd": "34869.7692419204775049", "time": 1609891200000, "date": "2021-01-06T00:00:00.000Z", }, { "priceUsd": "38041.0026368820979411", "time": 1609977600000, "date": "2021-01-07T00:00:00.000Z", }, { "priceUsd": "39821.5432664411153366", "time": 1610064000000, "date": "2021-01-08T00:00:00.000Z", }, ], "timestamp": 1618219315479, } def response(content, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = content return mock.patch("requests.get", return_value=response) class Source(unittest.TestCase): def test_get_latest_price(self): with response(content=response_assets_bitcoin): srcprice = coincap.Source().get_latest_price("bitcoin") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("60748.3135183678858890"), srcprice.price) self.assertEqual( datetime.datetime(2021, 4, 12) .replace(tzinfo=datetime.timezone.utc) .date(), srcprice.time.date(), ) self.assertEqual("USD", srcprice.quote_currency) def test_get_historical_price(self): with response(content=response_assets_bitcoin_historical): srcprice = coincap.Source().get_historical_price( "bitcoin", datetime.datetime(2021, 1, 6).replace(tzinfo=timezone) ) self.assertEqual(Decimal("34869.7692419204775049"), srcprice.price) self.assertEqual( datetime.datetime(2021, 1, 6) .replace(tzinfo=datetime.timezone.utc) .date(), srcprice.time.date(), ) self.assertEqual("USD", srcprice.quote_currency) def test_get_prices_series(self): with response(content=response_bitcoin_history): srcprices = coincap.Source().get_prices_series( "bitcoin", datetime.datetime(2021, 1, 1).replace(tzinfo=timezone), datetime.datetime(2021, 3, 20).replace(tzinfo=timezone), ) self.assertEqual(len(srcprices), 8) self.assertEqual(Decimal("29232.6707650537687673"), srcprices[0].price) self.assertEqual( datetime.datetime(2021, 1, 1) .replace(tzinfo=datetime.timezone.utc) .date(), srcprices[0].time.date(), ) self.assertEqual("USD", srcprices[0].quote_currency) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/coinmarketcap.py000066400000000000000000000041671475134044500222650ustar00rootroot00000000000000"""A source fetching cryptocurrency prices from Coinmarketcap. Valid tickers are in the form "XXX-YYY", such as "BTC-CHF". It requires a free api key which needs to be set in the environment variable "COINMARKETCAP_API_KEY" Here is the API documentation: https://coinmarketcap.com/api/documentation/v1/ """ from decimal import Decimal import re from os import environ import requests from dateutil.parser import parse from beanprice import source class CoinmarketcapApiError(ValueError): "An error from the CoinMarketCap API." def _parse_ticker(ticker): """Parse the base and quote currencies from the ticker. Args: ticker: A string, the symbol in XXX-YYY format. Returns: A pair of (base, quote) currencies. """ match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) if not match: raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') return match.groups() class Source(source.Source): def get_latest_price(self, ticker): symbol, base = _parse_ticker(ticker) headers = { "X-CMC_PRO_API_KEY": environ["COINMARKETCAP_API_KEY"], } params = { "symbol": symbol, "convert": base, } resp = requests.get( url="https://pro-api.coinmarketcap.com/v1/cryptocurrency/quotes/latest", params=params, headers=headers, ) if resp.status_code != requests.codes.ok: raise CoinmarketcapApiError( "Invalid response ({}): {}".format(resp.status_code, resp.text) ) data = resp.json() if data["status"]["error_code"] != 0: status = data["status"] raise CoinmarketcapApiError( "Invalid response ({}): {}".format( status["error_code"], status["error_message"] ) ) quote = data["data"][symbol]["quote"][base] price = Decimal(str(quote["price"])) date = parse(quote["last_updated"]) return source.SourcePrice(price, date, base) def get_historical_price(self, ticker, time): return None beanprice-2.0.0/beanprice/sources/coinmarketcap_test.py000066400000000000000000000042121475134044500233130ustar00rootroot00000000000000import unittest from decimal import Decimal from os import environ from unittest import mock import requests from beanprice import source from beanprice.sources import coinmarketcap def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class CoinmarketcapPriceFetcher(unittest.TestCase): def setUp(self): environ["COINMARKETCAP_API_KEY"] = "foo" def tearDown(self): del environ["COINMARKETCAP_API_KEY"] def test_error_invalid_ticker(self): with self.assertRaises(ValueError): coinmarketcap.Source().get_latest_price("INVALID") def test_error_network(self): with response("Foobar", 404): with self.assertRaises(ValueError): coinmarketcap.Source().get_latest_price("BTC-CHF") def test_error_request(self): contents = { "status": { "error_code": 2, "error_message": "foobar", } } with response(contents): with self.assertRaises(ValueError): coinmarketcap.Source().get_latest_price("BTC-CHF") def test_valid_response(self): contents = { "data": { "BTC": { "quote": { "CHF": { "price": 1234.56, "last_updated": "2018-08-09T21:56:28.000Z", } } } }, "status": { "error_code": 0, "error_message": "", }, } with response(contents): srcprice = coinmarketcap.Source().get_latest_price("BTC-CHF") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("1234.56"), srcprice.price) self.assertEqual("CHF", srcprice.quote_currency) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/eastmoneyfund.py000066400000000000000000000101121475134044500223110ustar00rootroot00000000000000""" A source fetching fund price(net value) from eastmoneyfund(天天基金) which is a chinese securities company. eastmoneyfund supports many kinds of fund, such as fixed income fund, ETF, etc. this script only supports specific fund which table's header is following: https://fundf10.eastmoney.com/F10DataApi.aspx?type=lsjz&code=377240. fixed income fund is not supported, likes: https://fundf10.eastmoney.com/F10DataApi.aspx?type=lsjz&code=040003 the API, as far as I know, is undocumented. Prices are denoted in CNY. Timezone information: the http API requests GMT+8, the function transfers timezone to GMT+8 automatically """ import datetime import re from decimal import Decimal import requests from beanprice import source # All of the easymoney funds are in CNY. CURRENCY = "CNY" TIMEZONE = datetime.timezone(datetime.timedelta(hours=+8), "Asia/Shanghai") headers = { "content-type": "application/json", "User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:22.0)" "Gecko/20100101 Firefox/22.0", } class EastMoneyFundError(ValueError): "An error from the EastMoneyFund API." UnsupportTickerError = EastMoneyFundError("header not match, dont support this ticker type") def parse_page(page): tr_re = re.compile(r"(.*?)") item_re = re.compile( r"(\d{4}-\d{2}-\d{2})(.*?)(.*?)" "(.*?)(.*?)(.*?)", re.X, ) header_match = re.compile( r"单位净值累计净值日增长率" "申购状态赎回状态.*?分红送配" ) table = tr_re.findall(page) if not header_match.match(table[0]): raise UnsupportTickerError try: table = [ ( datetime.datetime.fromisoformat(t[0]).replace(hour=15, tzinfo=TIMEZONE), Decimal(t[1]), ) for t in [item_re.match(x).groups() for x in table[1:]] ] except AttributeError: return None return table def get_price_series( ticker: str, time_begin: datetime.datetime, time_end: datetime.datetime ): base_url = "https://fundf10.eastmoney.com/F10DataApi.aspx" time_delta_day = (time_end - time_begin).days + 1 pages = time_delta_day // 30 + 1 res = [] for page in range(1, pages + 1): query = { "code": ticker, "page": str(page), "sdate": time_begin.astimezone(TIMEZONE).date().isoformat(), "edate": time_end.astimezone(TIMEZONE).date().isoformat(), "type": "lsjz", "per": str(30), } response = requests.get(base_url, params=query, headers=headers) if response.status_code != requests.codes.ok: raise EastMoneyFundError( f"Invalid response ({response.status_code}): {response.text}" ) price = parse_page(response.text) if price is None and page == 1: raise EastMoneyFundError( f"Invalid ticker {ticker} or " f"search day {time_begin.date().isoformat()}~{time_end.date().isoformat()}" ) if price is None: break res.extend(price) return res class Source(source.Source): def get_latest_price(self, ticker): end_time = datetime.datetime.now(TIMEZONE) begin_time = end_time - datetime.timedelta(days=10) prices = get_price_series(ticker, begin_time, end_time) last_price = prices[0] return source.SourcePrice(last_price[1], last_price[0], CURRENCY) def get_historical_price(self, ticker, time): prices = get_price_series(ticker, time - datetime.timedelta(days=10), time) last_price = prices[0] return source.SourcePrice(last_price[1], last_price[0], CURRENCY) def get_prices_series(self, ticker, time_begin, time_end): res = [ source.SourcePrice(x[1], x[0], CURRENCY) for x in get_price_series(ticker, time_begin, time_end) ] return sorted(res, key=lambda x: x.time) beanprice-2.0.0/beanprice/sources/eastmoneyfund_test.py000066400000000000000000000160511475134044500233600ustar00rootroot00000000000000import datetime import unittest from decimal import Decimal from unittest import mock from dateutil import tz import requests from beanprice.sources import eastmoneyfund from beanprice import source # ruff: noqa: E501,RUF001 CONTENTS = """ var apidata={ content:"
净值日期单位净值累计净值日增长率申购状态赎回状态分红送配
2020-10-095.18905.18904.11%开放申购开放赎回
2020-09-304.98404.98400.12%开放申购开放赎回
2020-09-294.97804.97801.14%开放申购开放赎回
2020-09-284.92204.92200.22%开放申购开放赎回
2020-09-254.91104.91100.88%开放申购开放赎回
2020-09-244.86804.8680-3.81%开放申购开放赎回
2020-09-235.06105.06102.41%开放申购开放赎回
2020-09-224.94204.9420-1.02%开放申购开放赎回
2020-09-214.99304.9930-1.29%开放申购开放赎回
2020-09-185.05805.05800.48%开放申购开放赎回
2020-09-175.03405.03400.60%开放申购开放赎回
2020-09-165.00405.0040-1.28%开放申购开放赎回
2020-09-155.06905.06901.06%开放申购开放赎回
2020-09-145.01605.01600.42%开放申购开放赎回
2020-09-114.99504.99503.39%开放申购开放赎回
2020-09-104.83104.8310-0.29%开放申购开放赎回
",records:16,pages:1,curpage:1};\ """ UNSUPPORT_CONTENT = """ var apidata={ content:"
净值日期每万份收益7日年化收益率（%）申购状态赎回状态分红送配
2020-09-100.42301.5730%开放申购开放赎回
",records:1,pages:1,curpage:1};""" def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = contents return mock.patch("requests.get", return_value=response) class EastMoneyFundFetcher(unittest.TestCase): def test_error_network(self): with response(None, 404): with self.assertRaises(ValueError): eastmoneyfund.get_price_series( "377240", datetime.datetime.now(), datetime.datetime.now() ) def test_unsupport_page(self): with response(UNSUPPORT_CONTENT): with self.assertRaises(ValueError) as exc: eastmoneyfund.get_price_series( "377240", datetime.datetime.now(), datetime.datetime.now() ) self.assertEqual(eastmoneyfund.UnsupportTickerError, exc.exception) def test_latest_price(self): with response(CONTENTS): srcprice = eastmoneyfund.Source().get_latest_price("377240") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("5.1890"), srcprice.price) self.assertEqual("CNY", srcprice.quote_currency) def test_historical_price(self): with response(CONTENTS): time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) srcprice = eastmoneyfund.Source().get_historical_price("377240", time) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("5.1890"), srcprice.price) self.assertEqual("CNY", srcprice.quote_currency) self.assertEqual( datetime.datetime(2020, 10, 9, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), srcprice.time, ) def test_get_prices_series(self): with response(CONTENTS): time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) srcprice = eastmoneyfund.Source().get_prices_series( "377240", time - datetime.timedelta(days=10), time ) self.assertIsInstance(srcprice, list) self.assertIsInstance(srcprice[-1], source.SourcePrice) self.assertEqual(Decimal("5.1890"), srcprice[-1].price) self.assertEqual("CNY", srcprice[-1].quote_currency) self.assertEqual( datetime.datetime(2020, 10, 9, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), srcprice[-1].time, ) self.assertIsInstance(srcprice[0], source.SourcePrice) self.assertEqual(Decimal("4.8310"), srcprice[0].price) self.assertEqual("CNY", srcprice[0].quote_currency) self.assertEqual( datetime.datetime(2020, 9, 10, 15, 0, 0, tzinfo=eastmoneyfund.TIMEZONE), srcprice[0].time, ) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/ecbrates.py000066400000000000000000000105361475134044500212320ustar00rootroot00000000000000"""A source fetching exchange rates using European Central Bank's datasets This source leverages daily avarage rates to/from EUR. For other currency pairs the final rate is derived by dividing rates to/from EUR. Valid tickers are in the form "XXX-YYY", such as "EUR-CHF", which denotes rate EUR->CHF Here is the API documentation: https://data.ecb.europa.eu/help/api/overview Timezone information: Input and output datetimes are specified via UTC timestamps. """ from decimal import Decimal, getcontext import re import csv from io import StringIO from dateutil.tz import tz from dateutil.parser import parse import requests from beanprice import source class ECBRatesError(ValueError): "An error from the ECB Rates." def _parse_ticker(ticker): """Parse the base and quote currencies from the ticker. Args: ticker: A string, the symbol in XXX-YYY format. Returns: A pair of (base, quote) currencies. """ match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) if not match: raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') return match.groups() def _get_rate_EUR_to_CCY(currency, date): # Call API symbol = f"D.{currency}.EUR.SP00.A" params = {"format": "csvdata", "detail": "full", "lastNObservations": 1} if date is not None: params["endPeriod"] = date url = f"https://data-api.ecb.europa.eu/service/data/EXR/{symbol}" response = requests.get(url, params=params) if response.status_code != requests.codes.ok: raise ECBRatesError( f"Invalid response ({response.status_code}): {response.text}" ) # Parse results to a DictReader iterator results = csv.DictReader(StringIO(response.text)) # Retrieve exchange rate try: observation = next(results) except StopIteration: # When there's no data for a given date, an empty string is returned return None, None, None else: # Checking only the first observation and raising errors if there's a date mismatch rate = observation.get("OBS_VALUE") obs_date = observation.get("TIME_PERIOD") decimals = observation.get("DECIMALS") precision = int(decimals) + len(rate.split(".")[0].lstrip("0")) return Decimal(rate), obs_date, precision def _get_quote(ticker, date): base, symbol = _parse_ticker(ticker) if base == symbol: raise ECBRatesError( f"Base currency {base} must be different than symbol currency {symbol}" ) # Get EUR rates by calling the API (or use defaults) if base == "EUR" and symbol != "EUR": eur_to_symbol, symbol_rate_date, symbol_rate_precision = _get_rate_EUR_to_CCY( symbol, date ) eur_to_base = Decimal(1) base_rate_date = symbol_rate_date base_rate_precision = 28 elif base != "EUR" and symbol == "EUR": eur_to_base, base_rate_date, base_rate_precision = _get_rate_EUR_to_CCY( base, date ) eur_to_symbol = Decimal(1) symbol_rate_date = base_rate_date symbol_rate_precision = 28 else: eur_to_base, base_rate_date, base_rate_precision = _get_rate_EUR_to_CCY( base, date ) eur_to_symbol, symbol_rate_date, symbol_rate_precision = _get_rate_EUR_to_CCY( symbol, date ) # Raise error if retrieved subrates for differnt dates if base_rate_date != symbol_rate_date: raise ECBRatesError( f"Subrates for different dates: ({base}, {base_rate_date}) \ vs. ({symbol}, {symbol_rate_date})" ) # Calculate base -> symbol if eur_to_symbol is None or eur_to_base is None: raise ECBRatesError( f"At least one of the subrates returned None: \ (EUR{symbol}: {eur_to_symbol}, EUR{base}: {eur_to_base})" ) # Derive precision from sunrates (must be at least 5) minimal_precision = 5 getcontext().prec = max( minimal_precision, min(base_rate_precision, symbol_rate_precision) ) price = eur_to_symbol / eur_to_base time = parse(base_rate_date).replace(tzinfo=tz.tzutc()) return source.SourcePrice(price, time, symbol) class Source(source.Source): def get_latest_price(self, ticker): return _get_quote(ticker, None) def get_historical_price(self, ticker, time): return _get_quote(ticker, time.date().isoformat()) beanprice-2.0.0/beanprice/sources/ecbrates_test.py000066400000000000000000000064771475134044500223020ustar00rootroot00000000000000import unittest from datetime import datetime from decimal import Decimal from unittest import mock import requests from dateutil import tz from beanprice import source from beanprice.sources import ecbrates ECB_CSV = """KEY,FREQ,CURRENCY,CURRENCY_DENOM,EXR_TYPE,EXR_SUFFIX,TIME_PERIOD,OBS_VALUE,OBS\ _STATUS,OBS_CONF,OBS_PRE_BREAK,OBS_COM,TIME_FORMAT,BREAKS,COLLECTION,COMPILING_ORG,DISS_ORG\ ,DOM_SER_IDS,PUBL_ECB,PUBL_MU,PUBL_PUBLIC,UNIT_INDEX_BASE,COMPILATION,COVERAGE,DECIMALS,NAT\ _TITLE,SOURCE_AGENCY,SOURCE_PUB,TITLE,TITLE_COMPL,UNIT,UNIT_MULT EXR.D.SEK.EUR.SP00.A,D,SEK,EUR,SP00,A,2024-12-24,11.5335,A,F,,,P1D,,A,,,,,,,,,,4,,4F0,,Euro\ /Swedish krona,"ECB reference exchange rate, Euro/Swedish krona, 2:15 pm (C.E.T.)",SEK,0 """ ECB_CSV_HIST = """KEY,FREQ,CURRENCY,CURRENCY_DENOM,EXR_TYPE,EXR_SUFFIX,TIME_PERIOD,OBS_VALU\ E,OBS_STATUS,OBS_CONF,OBS_PRE_BREAK,OBS_COM,TIME_FORMAT,BREAKS,COLLECTION,COMPILING_ORG,DIS\ S_ORG,DOM_SER_IDS,PUBL_ECB,PUBL_MU,PUBL_PUBLIC,UNIT_INDEX_BASE,COMPILATION,COVERAGE,DECIMAL\ S,NAT_TITLE,SOURCE_AGENCY,SOURCE_PUB,TITLE,TITLE_COMPL,UNIT,UNIT_MULT EXR.D.SEK.EUR.SP00.A,D,SEK,EUR,SP00,A,2024-12-06,11.523,A,F,,,P1D,,A,,,,,,,,,,4,,4F0,,Euro/\ Swedish krona,"ECB reference exchange rate, Euro/Swedish krona, 2:15 pm (C.E.T.)",SEK,0 """ def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a CSV response.""" response = mock.Mock() response.status_code = status_code response.text = contents return mock.patch("requests.get", return_value=response) class ECBRatesErrorFetcher(unittest.TestCase): def test_error_invalid_ticker(self): with self.assertRaises(ValueError) as exc: ecbrates.Source().get_latest_price("INVALID") def test_error_network(self): with response("Foobar", 404): with self.assertRaises(ValueError) as exc: ecbrates.Source().get_latest_price("EUR-SEK") def test_empty_response(self): with response("", 200): with self.assertRaises(ecbrates.ECBRatesError) as exc: ecbrates.Source().get_latest_price("EUR-SEK") def test_valid_response(self): contents = ECB_CSV with response(contents): srcprice = ecbrates.Source().get_latest_price("EUR-SEK") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("11.5335"), srcprice.price) self.assertEqual("SEK", srcprice.quote_currency) self.assertIsInstance(srcprice.time, datetime) self.assertEqual( datetime(2024, 12, 24, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time ) def test_historical_price(self): time = datetime(2024, 12, 6, 16, 0, 0, tzinfo=tz.tzlocal()).astimezone( tz.tzutc() ) contents = ECB_CSV_HIST with response(contents): srcprice = ecbrates.Source().get_historical_price("EUR-SEK", time) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("11.523"), srcprice.price) self.assertEqual("SEK", srcprice.quote_currency) self.assertIsInstance(srcprice.time, datetime) self.assertEqual( datetime(2024, 12, 6, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time ) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/iex.py000066400000000000000000000035561475134044500202330ustar00rootroot00000000000000"""Fetch prices from the IEX 1.0 public API. This is a really fantastic exchange API with a lot of relevant information. Timezone information: There is currency no support for historical prices. The output datetime is provided as a UNIX timestamp. """ __copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime from decimal import Decimal from dateutil import tz import requests from beanprice import source class IEXError(ValueError): "An error from the IEX API." def fetch_quote(ticker): """Fetch the latest price for the given ticker.""" url = "https://api.iextrading.com/1.0/tops/last?symbols={}".format(ticker.upper()) response = requests.get(url) if response.status_code != requests.codes.ok: raise IEXError( "Invalid response ({}): {}".format(response.status_code, response.text) ) results = response.json() if len(results) != 1: raise IEXError("Invalid number of responses from IEX: {}".format(response.text)) result = results[0] price = Decimal(result["price"]).quantize(Decimal("0.01")) # IEX is American markets. us_timezone = tz.gettz("America/New_York") time = datetime.datetime.fromtimestamp(result["time"] / 1000) time = time.astimezone(us_timezone) # As far as can tell, all the instruments on IEX are priced in USD. return source.SourcePrice(price, time, "USD") class Source(source.Source): "IEX API price extractor." def get_latest_price(self, ticker): """See contract in beanprice.source.Source.""" return fetch_quote(ticker) def get_historical_price(self, ticker, time): """See contract in beanprice.source.Source.""" raise NotImplementedError( "This is now implemented at https://iextrading.com/developers/docs/#hist and " "needs to be added here." ) beanprice-2.0.0/beanprice/sources/iex_test.py000066400000000000000000000032631475134044500212650ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime import unittest from unittest import mock from decimal import Decimal from dateutil import tz import requests from beanprice import date_utils from beanprice import source from beanprice.sources import iex def response(contents, status_code=requests.codes.ok): """Produce a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class IEXPriceFetcher(unittest.TestCase): def test_error_network(self): with response(None, 404): with self.assertRaises(ValueError) as exc: iex.fetch_quote("AAPL") self.assertRegex(exc.message, "premium") def _test_valid_response(self): contents = [{"symbol": "HOOL", "price": 183.61, "size": 100, "time": 1590177596030}] with response(contents): srcprice = iex.fetch_quote("HOOL") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("183.61"), srcprice.price) self.assertEqual( datetime.datetime(2020, 5, 22, 19, 59, 56, 30000, tzinfo=tz.tzutc()), srcprice.time.astimezone(tz.tzutc()), ) self.assertEqual("USD", srcprice.quote_currency) def test_valid_response(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._test_valid_response() if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/oanda.py000066400000000000000000000100741475134044500205210ustar00rootroot00000000000000"""A source fetching currency prices from OANDA. Valid tickers are in the form "XXX_YYY", such as "EUR_USD". Here is the API documentation: https://developer.oanda.com/rest-live/rates/ For example: https://api-fxtrade.oanda.com/v1/candles?instrument=EUR_USD&granularity=D&start=2016-03-27T00%3A00%3A00Z&end=2016-04-04T00%3A00%3A00Z&candleFormat=midpoint Timezone information: Input and output datetimes are specified via UTC timestamps. """ __copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import re import datetime import json import logging from urllib import parse from decimal import Decimal from dateutil import tz from beanprice import source from beanprice import net_utils URL = "https://api-fxtrade.oanda.com/v1/candles" def _get_currencies(ticker): """Parse the base and quote currencies from the ticker. Args: ticker: A string, the symbol in XXX_YYY format. Returns: A pair of (base, quote) currencies. """ match = re.match("([A-Z]+)_([A-Z]+)$", ticker) if not match: return None, None return match.groups() def _fetch_candles(params): """Fetch the given URL from OANDA and return a list of (utc-time, price). Args: params: A dict of URL params values. Returns: A sorted list of (time, price) points. """ url = "?".join((URL, parse.urlencode(sorted(params.items())))) logging.info("Fetching '%s'", url) # Fetch the data. response = net_utils.retrying_urlopen(url) if response is None: return None data_string = response.read().decode("utf-8") # Parse it. data = json.loads(data_string, parse_float=Decimal) try: # Find the candle with the latest time before the given time we're searching # for. time_prices = [] candles = sorted(data["candles"], key=lambda candle: candle["time"]) for candle in candles: candle_dt_utc = datetime.datetime.strptime( candle["time"], r"%Y-%m-%dT%H:%M:%S.%fZ" ).replace(tzinfo=tz.tzutc()) candle_price = Decimal(candle["openMid"]) time_prices.append((candle_dt_utc, candle_price)) except KeyError: logging.error("Unexpected response data: %s", data) return None return sorted(time_prices) def _fetch_price(params_dict, time): """Fetch a price from OANDA using the given parameters.""" ticker = params_dict["instrument"] _, quote_currency = _get_currencies(ticker) if quote_currency is None: logging.error("Invalid price source ticker '%s'; must be like 'EUR_USD'", ticker) return time_prices = _fetch_candles(params_dict) if not time_prices: logging.error("No prices returned.") return # Get all the prices before and on the same date and find the latest. sorted_prices = [item for item in time_prices if item[0] <= time] if not sorted_prices: logging.error("No prices matched.") return time, price = sorted_prices[-1] return source.SourcePrice(price, time, quote_currency) class Source(source.Source): "OANDA price source extractor." def get_latest_price(self, ticker): """See contract in beanprice.source.Source.""" time = datetime.datetime.now(tz.tzutc()) params_dict = { "instrument": ticker, "granularity": "S5", # Every two hours. "count": "10", "candleFormat": "midpoint", } return _fetch_price(params_dict, time) def get_historical_price(self, ticker, time): """See contract in beanprice.source.Source.""" time = time.astimezone(tz.tzutc()) query_interval_begin = time - datetime.timedelta(days=5) query_interval_end = time + datetime.timedelta(days=1) params_dict = { "instrument": ticker, "granularity": "H2", # Every two hours. "candleFormat": "midpoint", "start": query_interval_begin.isoformat("T"), "end": query_interval_end.isoformat("T"), } return _fetch_price(params_dict, time) beanprice-2.0.0/beanprice/sources/oanda_test.py000066400000000000000000000165301475134044500215630ustar00rootroot00000000000000"""Test for price extractor of OANDA.""" __copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import os import time import datetime import unittest from unittest import mock from decimal import Decimal from dateutil import tz from beanprice import date_utils from beanprice import net_utils from beanprice import source from beanprice.sources import oanda UTC = tz.tzutc() def response(code, contents=None): urlopen = mock.MagicMock(return_value=None) if isinstance(contents, str): response = mock.MagicMock() response.read = mock.MagicMock(return_value=contents.encode("utf-8")) response.getcode = mock.MagicMock(return_value=200) urlopen.return_value = response return mock.patch.object(net_utils, "retrying_urlopen", urlopen) class TestOandaMisc(unittest.TestCase): def test_get_currencies(self): self.assertEqual(("USD", "CAD"), oanda._get_currencies("USD_CAD")) def test_get_currencies_invalid(self): self.assertEqual((None, None), oanda._get_currencies("USDCAD")) class TimezoneTestBase: def setUp(self): tz_value = "Europe/Berlin" self.tz_old = os.environ.get("TZ", None) os.environ["TZ"] = tz_value time.tzset() def tearDown(self): if self.tz_old is None: del os.environ["TZ"] else: os.environ["TZ"] = self.tz_old time.tzset() class TestOandaFetchCandles(TimezoneTestBase, unittest.TestCase): @response(404) def test_null_response(self): self.assertIs(None, oanda._fetch_candles({})) @response( 200, """ { "instrument" : "USD_CAD", "granularity" : "S5" } """, ) def test_key_error(self): self.assertIs(None, oanda._fetch_candles({})) @response( 200, """ { "instrument" : "USD_CAD", "granularity" : "S5", "candles" : [ { "time" : "2017-01-23T00:45:15.000000Z", "openMid" : 1.330115, "highMid" : 1.33012, "lowMid" : 1.33009, "closeMid" : 1.33009, "volume" : 9, "complete" : true }, { "time" : "2017-01-23T00:45:20.000000Z", "openMid" : 1.330065, "highMid" : 1.330065, "lowMid" : 1.330065, "closeMid" : 1.330065, "volume" : 1, "complete" : true } ] } """, ) def test_valid(self): self.assertEqual( [ ( datetime.datetime(2017, 1, 23, 0, 45, 15, tzinfo=UTC), Decimal("1.330115"), ), ( datetime.datetime(2017, 1, 23, 0, 45, 20, tzinfo=UTC), Decimal("1.330065"), ), ], oanda._fetch_candles({}), ) class TestOandaGetLatest(unittest.TestCase): def setUp(self): self.fetcher = oanda.Source() def test_invalid_ticker(self): srcprice = self.fetcher.get_latest_price("NOTATICKER") self.assertIsNone(srcprice) def test_no_candles(self): with mock.patch.object(oanda, "_fetch_candles", return_value=None): self.assertEqual(None, self.fetcher.get_latest_price("USD_CAD")) def _test_valid(self): candles = [ (datetime.datetime(2017, 1, 21, 0, 45, 15, tzinfo=UTC), Decimal("1.330115")), (datetime.datetime(2017, 1, 21, 0, 45, 20, tzinfo=UTC), Decimal("1.330065")), ] with mock.patch.object(oanda, "_fetch_candles", return_value=candles): srcprice = self.fetcher.get_latest_price("USD_CAD") # Latest price, with current time as time. self.assertEqual( source.SourcePrice( Decimal("1.330065"), datetime.datetime(2017, 1, 21, 0, 45, 20, tzinfo=UTC), "CAD", ), srcprice, ) def test_valid(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._test_valid() class TestOandaGetHistorical(TimezoneTestBase, unittest.TestCase): def setUp(self): self.fetcher = oanda.Source() super().setUp() def test_invalid_ticker(self): srcprice = self.fetcher.get_latest_price("NOTATICKER") self.assertIsNone(srcprice) def test_no_candles(self): with mock.patch.object(oanda, "_fetch_candles", return_value=None): self.assertEqual(None, self.fetcher.get_latest_price("USD_CAD")) def _check_valid(self, query_date, out_time, out_price): candles = [ (datetime.datetime(2017, 1, 21, 0, 0, 0, tzinfo=UTC), Decimal("1.3100")), (datetime.datetime(2017, 1, 21, 8, 0, 0, tzinfo=UTC), Decimal("1.3300")), (datetime.datetime(2017, 1, 21, 16, 0, 0, tzinfo=UTC), Decimal("1.3500")), (datetime.datetime(2017, 1, 22, 0, 0, 0, tzinfo=UTC), Decimal("1.3700")), (datetime.datetime(2017, 1, 22, 8, 0, 0, tzinfo=UTC), Decimal("1.3900")), (datetime.datetime(2017, 1, 22, 16, 0, 0, tzinfo=UTC), Decimal("1.4100")), (datetime.datetime(2017, 1, 23, 0, 0, 0, tzinfo=UTC), Decimal("1.4300")), (datetime.datetime(2017, 1, 23, 8, 0, 0, tzinfo=UTC), Decimal("1.4500")), (datetime.datetime(2017, 1, 23, 16, 0, 0, tzinfo=UTC), Decimal("1.4700")), ] with mock.patch.object(oanda, "_fetch_candles", return_value=candles): query_time = datetime.datetime.combine( query_date, time=datetime.time(16, 0, 0), tzinfo=UTC ) srcprice = self.fetcher.get_historical_price("USD_CAD", query_time) if out_time is not None: self.assertEqual(source.SourcePrice(out_price, out_time, "CAD"), srcprice) else: self.assertEqual(None, srcprice) def test_valid_same_date(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._check_valid( datetime.date(2017, 1, 22), datetime.datetime(2017, 1, 22, 16, 0, tzinfo=UTC), Decimal("1.4100"), ) def test_valid_before(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._check_valid( datetime.date(2017, 1, 23), datetime.datetime(2017, 1, 23, 16, 0, tzinfo=UTC), Decimal("1.4700"), ) def test_valid_after(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._check_valid(datetime.date(2017, 1, 20), None, None) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/quandl.py000066400000000000000000000113451475134044500207250ustar00rootroot00000000000000"""Fetch prices from Quandl's simple URL-based API. Quandl is a useful source of alternative data and it offers a simple REST API that serves CSV and JSON and XML formats. There's also a Python client library, but we specifically avoid using that here, in order to keep Beancount dependency-free. Many of the datasets are freely available, which is why this is included here. You can get information about the available databases and associated lists of symbols you can use here: https://www.quandl.com/search If you have a paid account and would like to be able to access the premium databases from the Quandl site, you can set QUANDL_API_KEY environment variable. Use the ":" format to refer to Quandl symbols. Note that their symbols are usually identified by "/". If Quandl's output for the symbol you're interested in doesn't contain the default "Adj. Close" or "Close" column, you may specify the column to use after an additional semicolon, e.g. "::". If the column name contains spaces, use underscores instead, in order to not collide with the general price source syntax, e.g. "LBMA:GOLD:USD_(PM)". (For now, this supports only the Time-Series API. There is also a Tables API, which could easily get integrated. We would just have to encode the 'datatable_code' and 'format' and perhaps other fields in the ticker name.) Timezone information: Input and output datetimes are limited to dates, and I believe the dates are presumed to live in the timezone of each particular data source. (It's unclear, not documented.) """ __copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import collections import datetime import re import os from decimal import Decimal from dateutil import tz import requests from beanprice import source class QuandlError(ValueError): "An error from the Quandl API." TickerSpec = collections.namedtuple("TickerSpec", "database dataset column") def parse_ticker(ticker): """Convert ticker to Quandl codes.""" if not re.match(r"[A-Z0-9]+:[A-Z0-9]+(:[^:; ]+)?$", ticker): raise ValueError('Invalid code. Use ":[:]" format.') split = ticker.split(":") if len(split) == 2: return TickerSpec(split[0], split[1], None) return TickerSpec(split[0], split[1], split[2].replace("_", " ")) def fetch_time_series(ticker, time=None): """Fetch""" # Create request payload. ticker_spec = parse_ticker(ticker) url = "https://www.quandl.com/api/v3/datasets/{}/{}.json".format( ticker_spec.database, ticker_spec.dataset ) payload = {"limit": 1} if time is not None: date = time.date() payload["start_date"] = (date - datetime.timedelta(days=10)).isoformat() payload["end_date"] = date.isoformat() # Add API key, if it is set in the environment. if "QUANDL_API_KEY" in os.environ: payload["api_key"] = os.environ["QUANDL_API_KEY"] # Fetch and process errors. response = requests.get(url, params=payload) if response.status_code != requests.codes.ok: raise QuandlError( "Invalid response ({}): {}".format(response.status_code, response.text) ) result = response.json() if "quandl_error" in result: raise QuandlError(result["quandl_error"]["message"]) # Parse result container. dataset = result["dataset"] column_names = dataset["column_names"] date_index = column_names.index("Date") if ticker_spec.column is not None: data_index = column_names.index(ticker_spec.column) else: try: data_index = column_names.index("Adj. Close") except ValueError: data_index = column_names.index("Close") data = dataset["data"][0] # Gather time and assume it's in UTC timezone (Quandl does not provide the # market's timezone). time = datetime.datetime.strptime(data[date_index], "%Y-%m-%d") time = time.replace(tzinfo=tz.tzutc()) # Gather price. # Quantize with the same precision default rendering of floats occur. price_float = data[data_index] price = Decimal(price_float) match = re.search(r"(\..*)", str(price_float)) if match: price = price.quantize(Decimal(match.group(1))) # Note: There is no currency information in the response (surprising). return source.SourcePrice(price, time, None) class Source(source.Source): "Quandl API price extractor." def get_latest_price(self, ticker): """See contract in beanprice.source.Source.""" return fetch_time_series(ticker) def get_historical_price(self, ticker, time): """See contract in beanprice.source.Source.""" return fetch_time_series(ticker, time) beanprice-2.0.0/beanprice/sources/quandl_test.py000066400000000000000000000154201475134044500217620ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2018-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime import unittest from unittest import mock from decimal import Decimal from dateutil import tz import requests from beanprice import date_utils from beanprice import source from beanprice.sources import quandl def response(contents, status_code=requests.codes.ok): """Produce a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class QuandlPriceFetcher(unittest.TestCase): def test_parse_ticker(self): # NOTE(pmarciniak): "LBMA:GOLD:USD (PM)" is a valid ticker in Quandl # requests, but since space is not allowed in price source syntax, we're # representing space with an underscore. self.assertEqual( quandl.TickerSpec("WIKI", "FB", None), quandl.parse_ticker("WIKI:FB") ) self.assertEqual( quandl.TickerSpec("LBMA", "GOLD", "USD (PM)"), quandl.parse_ticker("LBMA:GOLD:USD_(PM)"), ) for test in [ "WIKI/FB", "FB", "WIKI.FB", "WIKI,FB", "LBMA:GOLD:USD (PM)", "LBMA:GOLD:col:umn", ]: with self.assertRaises(ValueError): quandl.parse_ticker(test) def test_error_premium(self): contents = { "quandl_error": { "code": "QEPx05", "message": ( "You have attempted to view a premium database in " "anonymous mode, i.e., without providing a Quandl " "key. Please register for a free Quandl account, " "and then include your API key with your " "requests." ), } } with response(contents): with self.assertRaises(ValueError) as exc: quandl.fetch_time_series("WIKI:FB", None) self.assertRegex(exc.message, "premium") def test_error_subscription(self): contents = { "quandl_error": { "code": "QEPx04", "message": ( "You do not have permission to view this dataset. " "Please subscribe to this database to get " "access." ), } } with response(contents): with self.assertRaises(ValueError) as exc: quandl.fetch_time_series("WIKI:FB", None) self.assertRegex(exc.message, "premium") def test_error_network(self): with response(None, 404): with self.assertRaises(ValueError) as exc: quandl.fetch_time_series("WIKI:FB", None) self.assertRegex(exc.message, "premium") def _test_valid_response(self): contents = { "dataset": { "collapse": None, "column_index": None, "column_names": [ "Date", "Open", "High", "Low", "Close", "Volume", "Ex-Dividend", "Split Ratio", "Adj. Open", "Adj. High", "Adj. Low", "Adj. Close", "Adj. Volume", ], "data": [ [ "2018-03-27", 1063.9, 1064.54, 997.62, 1006.94, 2940957.0, 0.0, 1.0, 1063.9, 1064.54, 997.62, 1006.94, 2940957.0, ] ], "database_code": "WIKI", "database_id": 4922, "dataset_code": "GOOGL", "description": "This dataset has no description.", "end_date": "2018-03-27", "frequency": "daily", "id": 11304017, "limit": 1, "name": ( "Alphabet Inc (GOOGL) Prices, Dividends, Splits and " "Trading Volume" ), "newest_available_date": "2018-03-27", "oldest_available_date": "2004-08-19", "order": None, "premium": False, "refreshed_at": "2018-03-27T21:46:11.201Z", "start_date": "2004-08-19", "transform": None, "type": "Time Series", } } with response(contents): srcprice = quandl.fetch_time_series("WIKI:FB", None) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("1006.94"), srcprice.price) self.assertEqual( datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time.astimezone(tz.tzutc()), ) self.assertEqual(None, srcprice.quote_currency) def test_valid_response(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._test_valid_response() def test_non_standard_columns(self): contents = { "dataset": { "collapse": None, "column_index": None, "column_names": [ "Date", "USD (AM)", "USD (PM)", "GBP (AM)", "GBP (PM)", "EURO (AM)", "EURO (PM)", ], "data": [ ["2019-06-18", 1344.55, 1341.35, 1073.22, 1070.67, 1201.89, 1198.09] ], "end_date": "2019-06-18", "frequency": "daily", "order": None, "limit": 1, "start_date": "2019-06-08", "transform": None, } } with response(contents): srcprice = quandl.fetch_time_series("LBMA:GOLD:USD_(PM)", None) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("1341.35"), srcprice.price) self.assertEqual( datetime.datetime(2019, 6, 18, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time.astimezone(tz.tzutc()), ) self.assertEqual(None, srcprice.quote_currency) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/ratesapi.py000066400000000000000000000034441475134044500212520ustar00rootroot00000000000000"""A source fetching exchangerates from https://exchangerate.host. Valid tickers are in the form "XXX-YYY", such as "EUR-CHF". Here is the API documentation: https://api.frankfurter.app/ For example: https://api.frankfurter.app/latest?base=EUR&symbols=CHF Timezone information: Input and output datetimes are specified via UTC timestamps. """ from decimal import Decimal import re import requests from dateutil.tz import tz from dateutil.parser import parse from beanprice import source class RatesApiError(ValueError): "An error from the Rates API." def _parse_ticker(ticker): """Parse the base and quote currencies from the ticker. Args: ticker: A string, the symbol in XXX-YYY format. Returns: A pair of (base, quote) currencies. """ match = re.match(r"^(?P\w+)-(?P\w+)$", ticker) if not match: raise ValueError('Invalid ticker. Use "BASE-SYMBOL" format.') return match.groups() def _get_quote(ticker, date): """Fetch a exchangerate from ratesapi.""" base, symbol = _parse_ticker(ticker) params = { "base": base, "symbol": symbol, } response = requests.get(url="https://api.frankfurter.app/" + date, params=params) if response.status_code != requests.codes.ok: raise RatesApiError( "Invalid response ({}): {}".format(response.status_code, response.text) ) result = response.json() price = Decimal(str(result["rates"][symbol])) time = parse(result["date"]).replace(tzinfo=tz.tzutc()) return source.SourcePrice(price, time, symbol) class Source(source.Source): def get_latest_price(self, ticker): return _get_quote(ticker, "latest") def get_historical_price(self, ticker, time): return _get_quote(ticker, time.date().isoformat()) beanprice-2.0.0/beanprice/sources/ratesapi_test.py000066400000000000000000000040251475134044500223050ustar00rootroot00000000000000import datetime import unittest from decimal import Decimal from unittest import mock from dateutil import tz import requests from beanprice import source from beanprice.sources import ratesapi def response(contents, status_code=requests.codes.ok): """Return a context manager to patch a JSON response.""" response = mock.Mock() response.status_code = status_code response.text = "" response.json.return_value = contents return mock.patch("requests.get", return_value=response) class RatesapiPriceFetcher(unittest.TestCase): def test_error_invalid_ticker(self): with self.assertRaises(ValueError): ratesapi.Source().get_latest_price("INVALID") def test_error_network(self): with response("Foobar", 404): with self.assertRaises(ValueError): ratesapi.Source().get_latest_price("EUR-CHF") def test_valid_response(self): contents = { "base": "EUR", "rates": {"CHF": "1.2001"}, "date": "2019-04-20", } with response(contents): srcprice = ratesapi.Source().get_latest_price("EUR-CHF") self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("1.2001"), srcprice.price) self.assertEqual("CHF", srcprice.quote_currency) def test_historical_price(self): time = datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()) contents = { "base": "EUR", "rates": {"CHF": "1.2001"}, "date": "2018-03-27", } with response(contents): srcprice = ratesapi.Source().get_historical_price("EUR-CHF", time) self.assertIsInstance(srcprice, source.SourcePrice) self.assertEqual(Decimal("1.2001"), srcprice.price) self.assertEqual("CHF", srcprice.quote_currency) self.assertEqual( datetime.datetime(2018, 3, 27, 0, 0, 0, tzinfo=tz.tzutc()), srcprice.time ) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/tsp.py000066400000000000000000000104211475134044500202410ustar00rootroot00000000000000"""Fetch prices from US Government Thrift Savings Plan As of 7 July 2020, the Thrift Savings Plan (TSP) rolled out a new web site that has an API (instead of scraping a CSV). Unable to find docs on the API. A web directory listing with various tools is available at: https://secure.tsp.gov/components/CORS/ """ __copyright__ = "Copyright (C) 2020 Martin Blais" __license__ = "GNU GPLv2" import csv from collections import OrderedDict import datetime from decimal import Decimal import requests from beanprice import source # All of the TSP funds are in USD. CURRENCY = "USD" TIMEZONE = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") TSP_FUND_NAMES = [ "LInco", # 0 "L2025", # 1 "L2030", # 2 "L2035", # 3 "L2040", # 4 "L2045", # 5 "L2050", # 6 "L2055", # 7 "L2060", # 8 "L2065", # 9 "GFund", # 10 "FFund", # 11 "CFund", # 12 "SFund", # 13 "IFund", # 14 ] csv.register_dialect( "tsp", delimiter=",", quoting=csv.QUOTE_NONE, # NOTE(blais): This fails to import in 3.12 (and perhaps before). # quotechar='', lineterminator="\n", ) class TSPError(ValueError): "An error from the Thrift Savings Plan (TSP) API." def parse_tsp_csv(response: requests.models.Response) -> OrderedDict: """Parses a Thrift Savings Plan output CSV file. Function takes in a requests response and returns an OrderedDict with newest closing cost at front of OrderedDict. """ data = OrderedDict() text = response.iter_lines(decode_unicode=True) reader = csv.DictReader(text, dialect="tsp") for row in reader: # Date from TSP looks like "July 30. 2020" # There is indeed a period after the day of month. date = datetime.datetime.strptime(row["Date"], "%b %d. %Y") date = date.replace(hour=16, tzinfo=TIMEZONE) names = [ "L Income", "L 2025", "L 2030", "L 2035", "L 2040", "L 2045", "L 2050", "L 2055", "L 2060", "L 2065", "G Fund", "F Fund", "C Fund", "S Fund", "I Fund", ] data[date] = [ Decimal(row[name]) if row[name] else Decimal() for name in map(str.strip, names) ] return OrderedDict(sorted(data.items(), key=lambda t: t[0], reverse=True)) def parse_response(response: requests.models.Response) -> OrderedDict: """Process as response from TSP. Raises: TSPError: If there is an error in the response. """ if response.status_code != requests.codes.ok: raise TSPError("Error from TSP Parsing Status {}".format(response.status_code)) return parse_tsp_csv(response) class Source(source.Source): "US Thrift Savings Plan API Price Extractor" def get_latest_price(self, fund): """See contract in beanprice.source.Source.""" return self.get_historical_price(fund, datetime.datetime.now()) def get_historical_price(self, fund, time): """See contract in beanprice.source.Source.""" if requests is None: raise TSPError("You must install the 'requests' library.") if fund not in TSP_FUND_NAMES: raise TSPError( "Invalid TSP Fund Name '{}'. Valid Funds are:\n\t{}".format( fund, "\n\t".join(TSP_FUND_NAMES) ) ) url = "https://secure.tsp.gov/components/CORS/getSharePricesRaw.html" payload = { # Grabbing the last fourteen days of data in event the markets were closed. "startdate": (time - datetime.timedelta(days=14)).strftime("%Y%m%d"), "enddate": time.strftime("%Y%m%d"), "download": "0", "Lfunds": "1", "InvFunds": "1", } response = requests.get(url, params=payload) result = parse_response(response) trade_day = next(iter(result.items())) prices = trade_day[1] try: price = prices[TSP_FUND_NAMES.index(fund)] trade_time = trade_day[0] except KeyError as exc: raise TSPError("Invalid response from TSP: {}".format(repr(result))) from exc return source.SourcePrice(price, trade_time, CURRENCY) beanprice-2.0.0/beanprice/sources/tsp_test.py000066400000000000000000000150371475134044500213100ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2020 Martin Blais" __license__ = "GNU GPLv2" import datetime import textwrap import unittest from decimal import Decimal from unittest import mock import requests from beanprice.sources import tsp CURRENT_DATA = ( "Date,L Income,L 2025,L 2030,L 2035,L 2040,L 2045," "L 2050,L 2055,L 2060,L 2065,G Fund,F Fund,C Fund,S Fund,I Fund\n" "Jul 2. 2020, 21.1910, 10.0404, 34.1084, 10.0531, 37.3328, 10.0617," " 21.6880, 10.0780, 10.0780, 10.0781, 16.4477, 20.9449, 46.2229, 53.2754, 29.5401\n" "Jul 6. 2020, 21.2689, 10.1224, 34.4434, 10.1610, 37.7690, 10.1871," " 21.9759, 10.2392, 10.2392, 10.2393, 16.4490, 20.9555, 46.9567, 54.0427, 30.0523\n" "Jul 7. 2020, 21.2148, 10.0628, 34.1982, 10.0819, 37.4478, 10.0946," " 21.7627, 10.1173, 10.1174, 10.1175, 16.4493, 20.9946, 46.4489, 53.3353, 29.6545\n" "Jul 8. 2020, 21.2424, 10.0924, 34.3196, 10.1213, 37.6071, 10.1407," " 21.8687, 10.1771, 10.1772, 10.1773, 16.4496, 20.9960, 46.8131, 53.9192, 29.6863\n" "Jul 9. 2020, 21.2175, 10.0632, 34.1992, 10.0822, 37.4481, 10.0946," " 21.7622, 10.1144, 10.1145, 10.1146, 16.4499, 21.0549, 46.5615, 53.3815, 29.5161\n" "Jul 10. 2020, 21.2562, 10.1058, 34.3736, 10.1387, 37.6769, 10.1607," " 21.9144, 10.2014, 10.2014, 10.2015, 16.4502, 21.0288, 47.0497, 54.1006, 29.6343\n" "Jul 13. 2020, 21.2263, 10.0723, 34.2353, 10.0937, 37.4937, 10.1074," " 21.7911, 10.1317, 10.1317, 10.1318, 16.4512, 21.0316, 46.6089, 53.0366, 29.7075\n" "Jul 14. 2020, 21.2898, 10.1398, 34.5110, 10.1829, 37.8542, 10.2115," " 22.0301, 10.2651, 10.2651, 10.2652, 16.4515, 21.0608, 47.2391, 53.8560, 30.0643\n" "Jul 15. 2020, 21.3513, 10.2067, 34.7862, 10.2723, 38.2174, 10.3170," " 22.2736, 10.4025, 10.4026, 10.4027, 16.4519, 21.0574, 47.6702, 55.2910, 30.4751" ) HISTORIC_DATA = ( "Date,L Income,L 2025,L 2030,L 2035,L 2040,L 2045," "L 2050,L 2055,L 2060,L 2065,G Fund,F Fund,C Fund,S Fund,I Fund\n" "Jun 5. 2020, 21.2498,, 34.4615,, 37.8091,, 22.0123,,,," "16.4390, 20.6864, 47.1062, 54.6393, 30.1182\n" "Jun 8. 2020, 21.3126,, 34.7342,, 38.1665,, 22.2501,,,," "16.4400, 20.7235, 47.6757, 55.8492, 30.4200\n" "Jun 9. 2020, 21.2740,, 34.5504,, 37.9228,, 22.0861,,,," "16.4403, 20.7657, 47.3065, 54.7837, 30.2166\n" "Jun 10. 2020, 21.2531,, 34.4425,, 37.7784,, 21.9877,,,," "16.4407, 20.8290, 47.0540, 53.8901, 30.1676\n" "Jun 11. 2020, 20.9895,, 33.2506,, 36.2150,, 20.9486,,,," "16.4410, 20.8675, 44.2870, 50.2699, 28.5702\n" "Jun 12. 2020, 21.0436,, 33.4985,, 36.5408,, 21.1659,,,," "16.4413, 20.8332, 44.8769, 51.2514, 28.8452\n" "Jun 15. 2020, 21.0834,, 33.6719,, 36.7683,, 21.3177,,,," "16.4423, 20.8369, 45.2527, 52.3220, 28.9501\n" "Jun 16. 2020, 21.1666,, 34.0373,, 37.2440,, 21.6317,,,," "16.4426, 20.8358, 46.1115, 53.3428, 29.4106\n" "Jun 17. 2020, 21.1605,, 34.0068,, 37.2029,, 21.6035,,,," "16.4429, 20.8416, 45.9447, 52.8271, 29.5535\n" "Jun 18. 2020, 21.1562,, 33.9827,, 37.1713,, 21.5824,,,," "16.4432, 20.8718, 45.9718, 52.9328, 29.3908\n" "Jun 19. 2020, 21.1354,, 33.8890,, 37.0491,, 21.5018,,,," "16.4435, 20.8742, 45.7171, 52.7196, 29.2879" ) class MockResponse: """A mock requests.Response object for testing.""" def __init__(self, contents, status_code=requests.codes.ok): self.status_code = status_code self._content = contents def iter_lines(self, decode_unicode=False): return iter(self._content.splitlines()) class TSPFinancePriceFetcher(unittest.TestCase): def test_get_latest_price_L2050(self): response = MockResponse(textwrap.dedent(CURRENT_DATA)) with mock.patch("requests.get", return_value=response): srcprice = tsp.Source().get_latest_price("L2050") self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("22.2736"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") self.assertEqual( datetime.datetime(2020, 7, 15, 16, 0, 0, tzinfo=timezone), srcprice.time ) self.assertEqual("USD", srcprice.quote_currency) def test_get_latest_price_SFund(self): response = MockResponse(textwrap.dedent(CURRENT_DATA)) with mock.patch("requests.get", return_value=response): srcprice = tsp.Source().get_latest_price("SFund") self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("55.2910"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") self.assertEqual( datetime.datetime(2020, 7, 15, 16, 0, 0, tzinfo=timezone), srcprice.time ) self.assertEqual("USD", srcprice.quote_currency) def test_get_historical_price(self): response = MockResponse(textwrap.dedent(HISTORIC_DATA)) with mock.patch("requests.get", return_value=response): srcprice = tsp.Source().get_historical_price( "CFund", time=datetime.datetime(2020, 6, 19) ) self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("45.7171"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") self.assertEqual( datetime.datetime(2020, 6, 19, 16, 0, 0, tzinfo=timezone), srcprice.time ) self.assertEqual("USD", srcprice.quote_currency) def test_get_historical_price_L2060(self): # This fund did not exist until 01 Jul 2020. Ensuring we get a Decimal(0.0) back. response = MockResponse(textwrap.dedent(HISTORIC_DATA)) with mock.patch("requests.get", return_value=response): srcprice = tsp.Source().get_historical_price( "L2060", time=datetime.datetime(2020, 6, 19) ) self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("0.0"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/New_York") self.assertEqual( datetime.datetime(2020, 6, 19, 16, 0, 0, tzinfo=timezone), srcprice.time ) self.assertEqual("USD", srcprice.quote_currency) def test_invalid_fund_latest(self): with self.assertRaises(tsp.TSPError): tsp.Source().get_latest_price("InvalidFund") def test_invalid_fund_historical(self): with self.assertRaises(tsp.TSPError): tsp.Source().get_historical_price("InvalidFund", time=datetime.datetime.now()) if __name__ == "__main__": unittest.main() beanprice-2.0.0/beanprice/sources/yahoo.py000066400000000000000000000150171475134044500205600ustar00rootroot00000000000000"""Fetch prices from Yahoo Finance's CSV API. As of late 2017, the older Yahoo finance API deprecated. In particular, the ichart endpoint is gone, and the download endpoint requires a cookie (which could be gotten - here's some documentation for that http://blog.bradlucas.com/posts/2017-06-02-new-yahoo-finance-quote-download-url/). We're using both the v7 and v8 APIs here, both of which are, as far as I can tell, undocumented: https://query1.finance.yahoo.com/v7/finance/quote https://query1.finance.yahoo.com/v8/finance/chart/SYMBOL Timezone information: Input and output datetimes are specified via UNIX timestamps, but the timezone of the particular market is included in the output. """ __copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" from datetime import datetime, timedelta, timezone from decimal import Decimal from typing import Any, Dict, List, Optional, Tuple, Union import requests from beanprice import source class YahooError(ValueError): "An error from the Yahoo API." def parse_response(response: requests.models.Response) -> Dict: """Process as response from Yahoo. Raises: YahooError: If there is an error in the response. """ json = response.json(parse_float=Decimal) content = next(iter(json.values())) if response.status_code != requests.codes.ok: raise YahooError("Status {}: {}".format(response.status_code, content["error"])) if len(json) != 1: raise YahooError( "Invalid format in response from Yahoo; many keys: {}".format( ",".join(json.keys()) ) ) if content["error"] is not None: raise YahooError("Error fetching Yahoo data: {}".format(content["error"])) if not content["result"]: raise YahooError("No data returned from Yahoo, ensure that the symbol is correct") return content["result"][0] # Note: Feel free to suggest more here via a PR. _MARKETS = { "us_market": "USD", "ca_market": "CAD", "ch_market": "CHF", } def parse_currency(result: Dict[str, Any]) -> Optional[str]: """Infer the currency from the result.""" if "market" not in result: return None return _MARKETS.get(result["market"], None) _DEFAULT_PARAMS = { "lang": "en-US", "corsDomain": "finance.yahoo.com", ".tsrc": "finance", } def get_price_series( ticker: str, time_begin: datetime, time_end: datetime ) -> Tuple[List[Tuple[datetime, Decimal]], str]: """Return a series of timestamped prices.""" if requests is None: raise YahooError("You must install the 'requests' library.") url = "https://query1.finance.yahoo.com/v8/finance/chart/{}".format(ticker) payload: Dict[str, Union[int, str]] = { "period1": int(time_begin.timestamp()), "period2": int(time_end.timestamp()), "interval": "1d", } payload.update(_DEFAULT_PARAMS) response = requests.get(url, params=payload, headers={"User-Agent": None}) result = parse_response(response) meta = result["meta"] tzone = timezone( timedelta(hours=meta["gmtoffset"] / 3600), meta["exchangeTimezoneName"] ) if "timestamp" not in result: raise YahooError( "Yahoo returned no data for ticker {} for time range {} - {}".format( ticker, time_begin, time_end ) ) timestamp_array = result["timestamp"] close_array = result["indicators"]["quote"][0]["close"] series = [ (datetime.fromtimestamp(timestamp, tz=tzone), Decimal(price)) for timestamp, price in zip(timestamp_array, close_array) if price is not None ] currency = result["meta"]["currency"] return series, currency class Source(source.Source): "Yahoo Finance CSV API price extractor." def get_latest_price(self, ticker: str) -> Optional[source.SourcePrice]: """See contract in beanprice.source.Source.""" session = requests.Session() session.headers.update( { "User-Agent": "Mozilla/5.0 (X11; Linux x86_64; rv:109.0) " "Gecko/20100101 Firefox/110.0" } ) # This populates the correct cookies in the session session.get("https://fc.yahoo.com") crumb = session.get("https://query1.finance.yahoo.com/v1/test/getcrumb").text url = "https://query1.finance.yahoo.com/v7/finance/quote" fields = ["symbol", "regularMarketPrice", "regularMarketTime"] payload = { "symbols": ticker, "fields": ",".join(fields), "exchange": "NYSE", "crumb": crumb, } payload.update(_DEFAULT_PARAMS) response = session.get(url, params=payload) try: result = parse_response(response) except YahooError as error: # The parse_response method cannot know which ticker failed, # but the user definitely needs to know which ticker failed! raise YahooError("%s (ticker: %s)" % (error, ticker)) from error try: price = Decimal(result["regularMarketPrice"]) tzone = timezone( timedelta(hours=result["gmtOffSetMilliseconds"] / 3600000), result["exchangeTimezoneName"], ) trade_time = datetime.fromtimestamp(result["regularMarketTime"], tz=tzone) except KeyError as exc: raise YahooError( "Invalid response from Yahoo: {}".format(repr(result)) ) from exc currency = parse_currency(result) return source.SourcePrice(price, trade_time, currency) def get_historical_price( self, ticker: str, time: datetime ) -> Optional[source.SourcePrice]: """See contract in beanprice.source.Source.""" # Get the latest data returned over the last 5 days. series, currency = get_price_series(ticker, time - timedelta(days=5), time) latest = None for data_dt, price in sorted(series): if data_dt >= time: break latest = data_dt, price if latest is None: raise YahooError("Could not find price before {} in {}".format(time, series)) return source.SourcePrice(price, data_dt, currency) def get_daily_prices( self, ticker: str, time_begin: datetime, time_end: datetime ) -> Optional[List[source.SourcePrice]]: """See contract in beanprice.source.Source.""" series, currency = get_price_series(ticker, time_begin, time_end) return [source.SourcePrice(price, time, currency) for time, price in series] beanprice-2.0.0/beanprice/sources/yahoo_test.py000066400000000000000000000324261475134044500216220ustar00rootroot00000000000000__copyright__ = "Copyright (C) 2015-2020 Martin Blais" __license__ = "GNU GPLv2" import datetime import json import textwrap import unittest from decimal import Decimal from unittest import mock import pytest from dateutil import tz import requests from beanprice import date_utils from beanprice.sources import yahoo class MockResponse: """A mock requests.Models.Response object for testing.""" def __init__(self, contents, status_code=requests.codes.ok): self.status_code = status_code self.contents = contents def json(self, **kwargs): return json.loads(self.contents, **kwargs) class YahooFinancePriceFetcher(unittest.TestCase): def _test_get_latest_price(self): response = MockResponse( textwrap.dedent(""" {"quoteResponse": {"error": null, "result": [{"esgPopulated": false, "exchange": "TOR", "exchangeDataDelayedBy": 15, "exchangeTimezoneName": "America/Toronto", "exchangeTimezoneShortName": "EDT", "fullExchangeName": "Toronto", "gmtOffSetMilliseconds": -14400000, "language": "en-US", "market": "ca_market", "marketState": "CLOSED", "quoteType": "ETF", "regularMarketPrice": 29.99, "regularMarketTime": 1522353589, "sourceInterval": 15, "symbol": "XSP.TO", "tradeable": false}]}} """) ) with mock.patch("requests.get", return_value=response): srcprice = yahoo.Source().get_latest_price("XSP.TO") self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("29.99"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/Toronto") self.assertEqual( datetime.datetime(2018, 3, 29, 15, 59, 49, tzinfo=timezone), srcprice.time ) self.assertEqual("CAD", srcprice.quote_currency) @pytest.mark.skip(reason="The mock.patch() call is incorrect, has not been updated.") def test_get_latest_price(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._test_get_latest_price() def _test_get_historical_price(self): response = MockResponse( textwrap.dedent(""" {"chart": {"error": null, "result": [{"indicators": {"adjclose": [{"adjclose": [29.236251831054688, 29.16683006286621, 29.196582794189453, 29.226333618164062]}], "quote": [{"close": [29.479999542236328, 29.40999984741211, 29.440000534057617, 29.469999313354492], "high": [29.510000228881836, 29.489999771118164, 29.469999313354492, 29.579999923706055], "low": [29.34000015258789, 29.350000381469727, 29.399999618530273, 29.43000030517578], "open": [29.360000610351562, 29.43000030517578, 29.43000030517578, 29.530000686645508], "volume": [160800, 118700, 98500, 227800]}]}, "meta": {"chartPreviousClose": 29.25, "currency": "CAD", "currentTradingPeriod": {"post": {"end": 1522702800, "gmtoffset": -14400, "start": 1522699200, "timezone": "EDT"}, "pre": {"end": 1522675800, "gmtoffset": -14400, "start": 1522670400, "timezone": "EDT"}, "regular": {"end": 1522699200, "gmtoffset": -14400, "start": 1522675800, "timezone": "EDT"}}, "dataGranularity": "1d", "exchangeName": "TOR", "exchangeTimezoneName": "America/Toronto", "firstTradeDate": 1018872000, "gmtoffset": -14400, "instrumentType": "ETF", "symbol": "XSP.TO", "timezone": "EDT", "validRanges": ["1d", "5d", "1mo", "3mo", "6mo", "1y", "2y", "5y", "10y", "ytd", "max"]}, "timestamp": [1509111000, 1509370200, 1509456600, 1509543000]}]}}""") ) with mock.patch("requests.get", return_value=response): srcprice = yahoo.Source().get_historical_price( "XSP.TO", datetime.datetime(2017, 11, 1, 16, 0, 0, tzinfo=tz.tzutc()) ) self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("29.469999313354492"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-4), "America/Toronto") self.assertEqual( datetime.datetime(2017, 11, 1, 9, 30, tzinfo=timezone), srcprice.time ) self.assertEqual("CAD", srcprice.quote_currency) def test_get_historical_price(self): for tzname in "America/New_York", "Europe/Berlin", "Asia/Tokyo": with date_utils.intimezone(tzname): self._test_get_historical_price() def test_parse_response_error_status_code(self): response = MockResponse( '{"quoteResponse": {"error": "Not supported", "result": [{}]}}', status_code=400 ) with self.assertRaises(yahoo.YahooError): yahoo.parse_response(response) def test_parse_response_error_invalid_format(self): response = MockResponse( """{"quoteResponse": {"error": null, "result": [{}]}, "chart": {"error": null, "result": [{}]}}""" ) with self.assertRaises(yahoo.YahooError): yahoo.parse_response(response) def test_parse_response_error_not_none(self): response = MockResponse( '{"quoteResponse": {"error": "Non-zero error", "result": [{}]}}' ) with self.assertRaises(yahoo.YahooError): yahoo.parse_response(response) def test_parse_response_empty_result(self): response = MockResponse('{"quoteResponse": {"error": null, "result": []}}') with self.assertRaises(yahoo.YahooError): yahoo.parse_response(response) def test_parse_response_no_timestamp(self): response = MockResponse( textwrap.dedent(""" {"chart": {"error": null, "result": [{"indicators": {"adjclose": [{}], "quote": [{}]}, "meta": {"chartPreviousClose": 29.25, "currency": "CAD", "currentTradingPeriod": {"post": {"end": 1522702800, "gmtoffset": -14400, "start": 1522699200, "timezone": "EDT"}, "pre": {"end": 1522675800, "gmtoffset": -14400, "start": 1522670400, "timezone": "EDT"}, "regular": {"end": 1522699200, "gmtoffset": -14400, "start": 1522675800, "timezone": "EDT"}}, "dataGranularity": "1d", "exchangeName": "TOR", "exchangeTimezoneName": "America/Toronto", "firstTradeDate": 1018872000, "gmtoffset": -14400, "instrumentType": "ETF", "symbol": "XSP.TO", "timezone": "EDT", "validRanges": ["1d", "5d", "1mo", "3mo", "6mo", "1y", "2y", "5y", "10y", "ytd", "max"]}}]}} """) ) with self.assertRaises(yahoo.YahooError): with mock.patch("requests.get", return_value=response): _ = yahoo.Source().get_historical_price( "XSP.TO", datetime.datetime(2017, 11, 1, 16, 0, 0, tzinfo=tz.tzutc()) ) def test_parse_null_prices_in_series(self): response = MockResponse( textwrap.dedent(""" {"chart": {"result":[ {"meta":{ "currency":"USD","symbol":"FBIIX", "exchangeName":"NAS","instrumentType":"MUTUALFUND", "firstTradeDate":1570714200,"regularMarketTime":1646053572, "gmtoffset":-18000,"timezone":"EST", "exchangeTimezoneName":"America/New_York", "regularMarketPrice":9.69,"chartPreviousClose":9.69, "priceHint":2, "currentTradingPeriod":{ "pre":{"timezone":"EST","start":1646038800,"end":1646058600,"gmtoffset":-18000}, "regular":{"timezone":"EST","start":1646058600,"end":1646082000,"gmtoffset":-18000}, "post":{"timezone":"EST","start":1646082000,"end":1646096400,"gmtoffset":-18000} }, "dataGranularity":"1d","range":"", "validRanges":["1mo","3mo","6mo","ytd","1y","2y","5y","10y","max"]}, "timestamp":[1645626600,1645713000,1645799400,1646058600], "indicators":{ "quote":[ {"open":[9.6899995803833,9.710000038146973,9.6899995803833,null], "low":[9.6899995803833,9.710000038146973,9.6899995803833,null], "high":[9.6899995803833,9.710000038146973,9.6899995803833,null], "volume":[0,0,0,null], "close":[9.6899995803833,9.710000038146973,9.6899995803833,null]} ],"adjclose":[ {"adjclose":[9.6899995803833,9.710000038146973,9.6899995803833,null]} ] }}],"error":null}} """) ) with mock.patch("requests.get", return_value=response): srcprice = yahoo.Source().get_historical_price( "XSP.TO", datetime.datetime(2022, 2, 28, 16, 0, 0, tzinfo=tz.tzutc()) ) self.assertTrue(isinstance(srcprice.price, Decimal)) self.assertEqual(Decimal("9.6899995803833"), srcprice.price) timezone = datetime.timezone(datetime.timedelta(hours=-5), "America/New_York") self.assertEqual( datetime.datetime(2022, 2, 25, 9, 30, tzinfo=timezone), srcprice.time ) self.assertEqual("USD", srcprice.quote_currency) if __name__ == "__main__": unittest.main() beanprice-2.0.0/bin/000077500000000000000000000000001475134044500142205ustar00rootroot00000000000000beanprice-2.0.0/bin/BUILD000066400000000000000000000007501475134044500150040ustar00rootroot00000000000000package(default_visibility = ["//visibility:public"]) # Note: The genrule targets rename the scripts to .py generated files. # Note: subpar does not yet support C extensions. genrule( name = "bean_price_py", srcs = ["bean-price"], outs = ["bean_price.py"], cmd = "cat $(locations :bean-price) > $@", ) py_binary( name = "bean_price", srcs = ["bean_price.py"], deps = [ "//beancount/prices:price", "//beancount:loader_with_plugins", ], ) beanprice-2.0.0/bin/bean-price000077500000000000000000000002221475134044500161470ustar00rootroot00000000000000#!/usr/bin/env python3 __copyright__ = "Copyright (C) 2013-2020 Martin Blais" __license__ = "GNU GPLv2" from beanprice.price import main; main() beanprice-2.0.0/etc/000077500000000000000000000000001475134044500142235ustar00rootroot00000000000000beanprice-2.0.0/etc/env000077500000000000000000000001771475134044500147460ustar00rootroot00000000000000#!/bin/sh # # My environment initialization for this project. USERPATH=$USERPATH:$PROJDIR/bin PYTHONPATH=$PYTHONPATH:$PROJDIR beanprice-2.0.0/experiments/000077500000000000000000000000001475134044500160135ustar00rootroot00000000000000beanprice-2.0.0/experiments/dividends/000077500000000000000000000000001475134044500177645ustar00rootroot00000000000000beanprice-2.0.0/experiments/dividends/download_dividends.py000077500000000000000000000045741475134044500242130ustar00rootroot00000000000000#!/usr/bin/env python3 """Download all dividends in a particular date interval.""" __copyright__ = "Copyright (C) 2020 Martin Blais" __license__ = "GNU GPLv2" from datetime import date as Date from decimal import Decimal from typing import List, Tuple import argparse import csv import datetime import io import pprint import dateutil.parser import requests def download_dividends( instrument: str, start_date: Date, end_date: Date ) -> List[Tuple[Date, Decimal]]: """Download a list of dividends issued over a time interval.""" tim = datetime.time() payload = { "period1": str(int(datetime.datetime.combine(start_date, tim).timestamp())), "period2": str(int(datetime.datetime.combine(end_date, tim).timestamp())), "interval": "1d", "events": "div", "includeAdjustedClose": "true", } template = " https://query1.finance.yahoo.com/v7/finance/download/{ticker}" url = template.format(ticker=instrument) resp = requests.get(url, params=payload) if not resp.ok: raise ValueError("Error fetching dividends: {}".format(resp.text)) rows = iter(csv.reader(io.StringIO(resp.text))) header = next(rows) if header != ["Date", "Dividends"]: raise ValueError( "Error fetching dividends: " "invalid response format: {}".format(header) ) dividends = [] for row in rows: date = datetime.datetime.strptime(row[0], "%Y-%m-%d").date() dividend = Decimal(row[1]) dividends.append((date, dividend)) return dividends def main(): """Top-level function.""" today = datetime.date.today() parser = argparse.ArgumentParser(description=__doc__.strip()) parser.add_argument("instrument", help="Yahoo!Finance code for financial instrument.") parser.add_argument( "start", action="store", type=lambda x: dateutil.parser.parse(x).date(), default=today.replace(year=today.year - 1), help="Start date of interval. Default is one year ago.", ) parser.add_argument( "end", action="store", type=lambda x: dateutil.parser.parse(x).date(), default=today, help="End date of interval. Default is today ago.", ) args = parser.parse_args() dividends = download_dividends(args.instrument, args.start, args.end) pprint.pprint(dividends) if __name__ == "__main__": main() beanprice-2.0.0/pyproject.toml000066400000000000000000000034221475134044500163650ustar00rootroot00000000000000[build-system] requires = ['setuptools'] build-backend = 'setuptools.build_meta' [project] name = 'beanprice' version = '2.0.0' description = 'Price quotes fetcher for Beancount' license = { file = 'COPYING' } readme = 'README.md' authors = [ { name = 'Martin Blais', email = 'blais@furius.ca' }, ] maintainers = [ { name = 'Martin Blais', email = 'blais@furius.ca' }, ] keywords = [ 'accounting', 'ledger', 'beancount', 'price' ] classifiers = [ 'License :: OSI Approved :: GNU General Public License v2 (GPLv2)', 'Programming Language :: Python :: 3 :: Only', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', 'Programming Language :: Python :: 3.12', 'Programming Language :: SQL', 'Topic :: Office/Business :: Financial :: Accounting', ] requires-python = '>= 3.9' dependencies = [ 'beancount >= 3.0.0', 'python-dateutil >= 2.6.0', 'requests >= 2.0', ] [project.scripts] bean-price = 'beanprice.price:main' [project.urls] homepage = 'https://github.com/beancount/beanprice' issues = 'https://github.com/beancount/beanprice/issues' [tool.setuptools.packages] find = {} [tool.coverage.run] branch = true [tool.coverage.report] exclude_also = [ 'if typing.TYPE_CHECKING:', ] [tool.ruff] line-length = 92 target-version = 'py39' [tool.ruff.lint] select = ['E', 'F', 'W', 'UP', 'B', 'C4', 'PL', 'RUF'] # TODO(blais): Review these ignores. ignore = [ 'RUF013', 'RUF005', 'PLW0603', 'UP014', 'UP031', 'B007', 'B905', 'C408', 'E731', 'PLR0911', 'PLR0912', 'PLR0913', 'PLR0915', 'PLR1714', 'PLR2004', 'PLW2901', 'RUF012', 'UP007', 'UP032', ] [tool.mypy] disable_error_code = ["import-untyped"] beanprice-2.0.0/requirements.txt000066400000000000000000000000661475134044500167360ustar00rootroot00000000000000beancount>=3.0.0 python-dateutil>=2.6.0 requests>=2.0 beanprice-2.0.0/requirements_dev.txt000066400000000000000000000000751475134044500175740ustar00rootroot00000000000000-r requirements.txt pylint==3.3.3 pytest==5.4.2 mypy==1.14.1 beanprice-2.0.0/setup.py000077500000000000000000000035111475134044500151650ustar00rootroot00000000000000#!/usr/bin/env python3 """Install script for beanprice.""" __copyright__ = "Copyright (C) 2008-2020 Martin Blais" __license__ = "GNU GPLv2" from setuptools import setup, find_packages setup( name="beanprice", version="2.0.0", description="Price quotes fetcher for Beancount", long_description=""" A script to fetch market data prices from various sources on the internet and render them for plain text accounting price syntax (and Beancount). """, license="GNU GPLv2 only", author="Martin Blais", author_email="blais@furius.ca", url="http://github.com/beancount/beanprice", download_url="https://github.com/beancount/beanprice", packages=find_packages(), install_requires=[ # Beancount library itself. "beancount>=3.0.0", # Testing support now uses the pytest module. "pytest", # This is required to parse dates from command-line options in a # loose, accepting format. Note that we use dateutil for timezone # database definitions as well, although it is inferior to pytz, but # because it can use the OS timezone database in the Windows # registry. See this article for context: # https://www.assert.cc/2014/05/25/which-python-time-zone-library.html # However, for creating offset timezones, we use the datetime.timezone # helper class because it is built-in. # Where this matters is for price source fetchers. # (Note: If pytz supported the Windows registry timezone information, # I would switch to that.) "python-dateutil", # This library is needed to make requests for price sources. "requests", ], entry_points={ "console_scripts": [ "bean-price = beanprice.price:main", ] }, python_requires=">=3.9", )