beancounter-0.8.10/0000755000000000000000000000000011405254411010770 5ustar beancounter-0.8.10/beancounter_schema_sqlite.txt0000644000000000000000000000454110005360224016736 0ustar create table beancounter ( version varchar(12) not null, data_last_updated timestamp with time zone ); create table cash ( name varchar(16) not null default '', value float default null, currency varchar(12) default null, type varchar(12) default null, owner varchar(16) default NULL, cost float default NULL, date date default NULL ); create table fxprices ( currency varchar(12) not null default '', date date default null, previous_close float4 default null, day_open float4 default null, day_low float4 default null, day_high float4 default null, day_close float4 default null, day_change float4 default null ); create table indices ( symbol varchar(12) not null default '', stockindex varchar(12) not null default '' ); create table portfolio ( symbol varchar(16) not null default '', shares float4 default null, currency varchar(12) default null, type varchar(16) default null, owner varchar(16) default null, cost float(4) default null, date date default null ); create table stockinfo ( symbol varchar(12) not null default '', name varchar(64) not null default '', exchange varchar(16) not null default '', capitalisation float4 default null, low_52weeks float4 default null, high_52weeks float4 default null, earnings float4 default null, dividend float4 default null, p_e_ratio float4 default null, avg_volume int4 default null, active boolean default TRUE ); create table stockprices ( symbol varchar(12) not null default '', date date default null, previous_close float4 default null, day_open float4 default null, day_low float4 default null, day_high float4 default null, day_close float4 default null, day_change float4 default null, bid float4 default null, ask float4 default null, volume int4 default null ); create unique index cash_pkey on cash (name,type,owner,date); create unique index fxprices_pkey on fxprices (currency, date); create unique index portfolio_pkey on portfolio (symbol, owner, date); create unique index stockinfo_pkey on stockinfo (symbol); create unique index stockprices_pkey on stockprices (symbol, date); beancounter-0.8.10/THANKS0000644000000000000000000000625711405253606011722 0ustar Andreas Wuest for suggestions and testing around early versions kristian orlopp missing Debian dependency on libdbd-pg-perl (if ODBC is not used) Horst.Timmermann@t-online.de (Horst Timmermann) for spotting a buglet in setup_beancounter in the 0.2.0 release Adrian Johnson for also spotting this, and a few more buglets leading to 0.2.1 Evelyn Mitchell for pointing out the need for additional $dbh->commit() calls and for the missing beancounter table insert in setup_beancounter Mathias Weidner for initial MySQL patches, numerous design discussions + suggestions and the patch to allow 'update' to either insert or update Max Bott for the bug report on quotes in company name's (DatabaseInfoData) Peter Kim for the update-in-batches-of-100 patch Ken Neighbors for two patched: correct documentation for beancounter, and adapt BeanCounter's backpopulate to a change at Yahoo in BeanCounter.pm Martin Weulersse for noticing a problem with the 'active' field under MySQL Bud Rogers for pointing out that datetime no longer works for PostgreSQL 7.3.* Thomas Walter for a bug report concerning a change in Yahoo!'s historical FX data as well as for a fine patch correcting a logic error in the SQL code in GetPriceData which affected the status report if securities had multiple positions across portfolios Robert A. Schmied for a thoughtful set of patches to beancounter, BeanCounter.pm setup_beancounter and update_beancounter. First Grand Slam winner :) And even more so for continued suggestions, testing and patches. Mike Castle for a patch with more consistent upper/lower case use in BeanCounter Joao Pedro for noting that password should be passwd in example.beancounterrc Phil Homewood for a patch dealing with 'times 100' scaling in FX rates Joao Antunes Costa for two patches testing for NAs in db updates, and ensuring firewall proxy information is set Kevin Kim for patches integrating beancounter with smtm Matthew Jurgens for a patch letting the non-US historical run batches of just under 200 data points to accomodate a Yahoo! restriction Oliver Muth for noticing that beancounter still used 'password' in the docs for the optional 'passwd' argument for db connections Rene Cunningham for a patch to prevent a division-by-zero if adjusted close was zero Pieter du Preez for a very comprehensive patch set that improved my code a lot Jesse Chu for noticing that closes weren't set to adj closes in backpopulating Doug Laidlaw for an updated rpm spec file Yanko Punchev for yet another fix to parsing daily capitalization Warren Thomposn for a patch correcting default currency behaviour David Jensen for passing on an Ubuntu bug report on an error in the manual page and all the kind folks who dropped a line to either report a bug, and/or say that they liked the program... beancounter-0.8.10/MANIFEST0000644000000000000000000000070210424554273012131 0ustar beancounter BeanCounter.pm beancounter_schema_mysql.txt beancounter_schema_postgresql.txt beancounter_schema_sqlite.txt beancounter_example.txt example.beancounterrc flip_symbol.sh Makefile.PL MANIFEST README README.Debian README.non-gnu setup_beancounter THANKS TODO update_beancounter t/01base.t contrib/getDiv contrib/schnapp contrib/beancounter.spec debian/changelog META.yml Module meta-data (added by MakeMaker) beancounter-0.8.10/update_beancounter0000755000000000000000000002006111405254334014570 0ustar #! /bin/bash -e # # update_beancounter --- Modify beancounter database # # Copyright (C) 2000 - 2006 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # $Id: update_beancounter,v 1.16 2006/03/22 04:15:45 edd Exp $ # If we really want to be independent of the underlying database, this # probably needs to written in Perl using DBI/DBD # # This once started as being PostgreSQL specific, and now also supports MySQL progname=$(basename $0) if [ "$USER" = "" ] then USER=$(whoami) fi VERSION='0.8.10' DB_SCHEMA='0.6.0' #DATABASE='testbean' DATABASE='beancounter' #PASSWORD= #DBSYSTEM='MySQL' DBSYSTEM='PostgreSQL' # -------------------------- Functions ---------------------------------- function usage_and_exit { cat< $DBCOMMAND" echo $query | $DBCOMMAND | grep -q $column rc=$? if [ "$rc" -ne 0 ]; then cmd="alter table $table add $column $spec;" echo $cmd | $DBCOMMAND ## echo "Running $cmd" fi set -e return 0 } function reindex_portfolio_table { if [ "$DBSYSTEM" = "MySQL" ]; then cmd="drop index portfolio_pkey on portfolio; create unique index portfolio_pkey on portfolio (symbol, owner, date);" else cmd="drop index portfolio_pkey; create unique index portfolio_pkey on portfolio (symbol, owner, date);" fi echo $cmd | $DBCOMMAND return 0 } # test if given column (argument 1) exists in a given table (argument 2) # and if so rename it to new name (argument 3) and spec (arg 4; mysql only) function rename_if_not_exists_column { oldname=$1 table=$2 column=$3 spec=$4 rc=0 set +e if [ "$DBSYSTEM" = "MySQL" ]; then query="show columns from $table" else query="\d $table" fi echo $query | $DBCOMMAND | grep -q "$column" rc=$? if [ "$rc" -ne 0 ]; then if [ "$DBSYSTEM" = "MySQL" ]; then # nothing to do as we only added this for 0.4.0 true else cmd="alter table $table rename $oldname to $column;" echo $cmd | $DBCOMMAND echo "Running $cmd" fi fi set -e return 0 } # -------------------------- Main --------------------------------------- while getopts ":mn:h" opt do case $opt in m) DBSYSTEM='MySQL' #echo "Now using $DBSYSTEM" ;; n) DATABASE=$OPTARG #echo "Now using database name $DATABASE" ;; h) usage_and_exit ;; ?) echo "Ignoring unknown argument, try '$progname -h' for help." ;; esac done echo "Examining database $DATABASE on $DBSYSTEM" if [ "$DBSYSTEM" = "MySQL" ] then # mysql(1) arguments -- you could add host, port, ... here if [ -z "$PASSWORD" ] then DBCOMMAND="mysql $DATABASE" else DBCOMMAND="mysql -p$PASSWORD $DATABASE" fi else if [ -z "$PASSWORD" ] then DBCOMMAND="psql -q $DATABASE" else DBCOMMAND="psql -q -W $PASSWORD $DATABASE" fi fi add_unless_exists_column type portfolio "varchar(16) default null" add_unless_exists_column owner portfolio "varchar(16) default null" add_unless_exists_column cost portfolio "float default null" add_unless_exists_column date portfolio "date default null" reindex_portfolio_table rename_if_not_exists_column change stockprices day_change "float default null" rename_if_not_exists_column change fxprices day_change "float default null" rename_if_not_exists_column index indices stockindex 'varchar(12) not null default ""' add_active_if_needed add_beancounter_table_if_needed set_version_to_current check_for_numeric_symbols echo "Done." exit 0 =head1 NAME update_beancounter - Convert older BeanCounter databases =head1 SYNOPSIS update_beancounter [-m] [-n NAME] [-h] =head1 DESCRIPTION B converts the databases used by B from an older release to the current one. =head1 OPTIONS -m Use MySQL as the backend over the default PostgreSQL -s name Use name as the database instead of B(1), B(1), B(1) =head1 AUTHOR Dirk Eddelbuettel edd@debian.org =cut beancounter-0.8.10/flip_symbol.sh0000755000000000000000000000047407603117670013665 0ustar #!/bin/sh if [ $# -ne 2 ] then echo "Usage: $0 from_symbol to_symbol" echo "Updates beancounter tables replacing the former with the latter." exit 1 fi for table in stockprices stockinfo portfolio indices do echo "update $table set symbol='$2' where symbol='$1';" | \ psql -e -d beancounter done beancounter-0.8.10/BeanCounter.html0000644000000000000000000002026510410150173014063 0ustar Finance::BeanCounter - Module for stock portfolio performance functions.

NAME

Finance::BeanCounter - Module for stock portfolio performance functions.

SYNOPSIS

 use Finance::BeanCounter;

DESCRIPTION

Finance::BeanCounter provides functions to download, store and analyse stock market data.

Downloads are available of current (or rather: 15 or 20 minute-delayed) price and company data as well as of historical price data. Both forms can be stored in an SQL database (for which we currently default to PostgreSQL though MySQL is supported as well; furthermore any database reachable by means of an ODBC connection should work).

Analysis currently consists of performance and risk analysis. Performance reports comprise a profit-and-loss (or 'p/l' in the lingo) report which can be run over arbitrary time intervals such as --prevdate 'friday six months ago' --date 'yesterday' -- in essence, whatever the wonderful Date::Manip module understands -- as well as dayendreport which defaults to changes in the last trading day. A risk report show parametric and non-parametric value-at-risk (VaR) estimates.

Most available functionality is also provided in the reference implementation beancounter, a convenient command-line script.

The API might change and evolve over time. The low version number really means to say that the code is not in its final form yet, but it has been in use for well over four years.

More documentation is in the Perl source code.

DATABASE LAYOUT

The easiest way to see the table design is to look at the content of the setup_beancounter script. It creates the five tables stockinfo, stockprices, fxprices, portfolio and indices. Note also that is supports the creation of database for both PostgreSQL and MySQL.

THE STOCKINFO TABLE

The stockinfo table contains general (non-price) information and is index by symbol:

            symbol              varchar(12) not null,
            name                varchar(64) not null,
            exchange            varchar(16) not null,
            capitalisation      float4,
            low_52weeks         float4,
            high_52weeks        float4,
            earnings            float4,
            dividend            float4,
            p_e_ratio           float4,
            avg_volume          int4

This table is updated by overwriting the previous content.

THE STOCKPRICES TABLE

The stockprices table contains (daily) price and volume information. It is indexed by both date and symbol:

            symbol              varchar(12) not null,
            date                date,
            previous_close      float4,
            day_open            float4,
            day_low             float4,
            day_high            float4,
            day_close           float4,
            day_change          float4,
            bid                 float4,
            ask                 float4,
            volume              int4

During updates, information is appended to this table.

THE FXPRICES TABLE

The fxprices table contains (daily) foreign exchange rates. It can be used to calculate home market values of foreign stocks:

            currency            varchar(12) not null,
            date                date,
            previous_close      float4,
            day_open            float4,
            day_low             float4,
            day_high            float4,
            day_close           float4,
            day_change          float4

Similar to the stockprices table, it is index on date and symbol.

THE STOCKPORTFOLIO TABLE

The portfolio table contains contains the holdings information:

            symbol              varchar(16) not null,
            shares              float4,
            currency            varchar(12),
            type                varchar(16),
            owner               varchar(16),
            cost                float(4),
            date                date

It is indexed on symbol,owner,date.

THE INDICES TABLE

The indices table links a stock symbol with one or several market indices:

            symbol              varchar(12) not null,
            stockindex          varchar(12) not null

BUGS

Finance::BeanCounter and beancounter are so fresh that there are only missing features :)

On a more serious note, this code (or its earlier predecessors) have been in use since the fall of 1998.

Known bugs or limitations are documented in TODO file in the source package.

SEE ALSO

beancounter.1, smtm.1, Finance::YahooQuote.3pm, LWP.3pm, Date::Manip.3pm

COPYRIGHT

Finance::BeanCounter.pm (c) 2000 -- 2006 by Dirk Eddelbuettel <edd@debian.org>

Updates to this program might appear at http://eddelbuettel.com/dirk/code/beancounter.html.

This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. There is NO warranty whatsoever.

The information that you obtain with this program may be copyrighted by Yahoo! Inc., and is governed by their usage license. See http://www.yahoo.com/docs/info/gen_disclaimer.html for more information.

ACKNOWLEDGEMENTS

The Finance::YahooQuote module by Dj Padzensky (on the web at http://www.padz.net/~djpadz/YahooQuote/) served as the backbone for data retrieval, and a guideline for the extension to the non-North American quotes which was already very useful for the real-time ticker http://eddelbuettel.com/dirk/code/smtm.html.

beancounter-0.8.10/META.yml0000644000000000000000000000114510376235214012250 0ustar # http://module-build.sourceforge.net/META-spec.html #XXXXXXX This is a prototype!!! It will change in the future!!! XXXXX# name: Finance-BeanCounter version: 0.8.4 version_from: installdirs: site requires: Date::Manip: 5.35 DBI: 1.16 Finance::YahooQuote: 0.2 HTML::Parser: 2.2 HTTP::Request: 1.23 LWP::UserAgent: 1.62 Statistics::Descriptive: 2.4 Text::ParseWords: 3.1 distribution_type: module generated_by: ExtUtils::MakeMaker version 6.17 beancounter-0.8.10/Makefile.PL0000644000000000000000000000357611405254334012761 0ustar # # Makefile for BeanCounter.pm # # BeanCounter.pm --- A stock portfolio performance monitoring tool # # Copyright (C) 2000 - 2005 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # $Id: Makefile.PL,v 1.3 2004/04/03 15:22:18 edd Exp edd $ # use ExtUtils::MakeMaker; WriteMakefile( 'NAME' => 'Finance::BeanCounter', 'VERSION' => '0.8.10', 'PREREQ_PM' => { "LWP::UserAgent" => 1.62, "HTTP::Request" => 1.23, "HTML::Parser" => 2.20, "Text::ParseWords" => 3.1, "Date::Manip" => 5.35, "DBI" => 1.16, ## and any one of these ## "DBD::Pg" => 0.93, ## "DBD::ODBC" => 0.28, ## "DBD::mysql" => 2.0416, ## "DBD::SQLite" => 1.0, # v3 ## "DBD::SQLite2"=> 0.33 "Statistics::Descriptive" => 2.4, "Finance::YahooQuote" => 0.20 }, ($] >= 5.005 ? ('ABSTRACT' => 'Stock portfolio performance monitoring tool', 'AUTHOR' => 'Dirk Eddelbuettel (edd@debian.org)') : () ), EXE_FILES => ['beancounter', 'setup_beancounter', 'update_beancounter'], 'dist' => { COMPRESS => "gzip -9f", SUFFIX => '.gz' } ); beancounter-0.8.10/beancounter_schema_postgresql.txt0000644000000000000000000000747110003626561017654 0ustar -- -- PostgreSQL database dump -- SET SESSION AUTHORIZATION 'postgres'; -- -- TOC entry 3 (OID 2200) -- Name: public; Type: ACL; Schema: -; Owner: postgres -- REVOKE ALL ON SCHEMA public FROM PUBLIC; REVOKE ALL ON SCHEMA public FROM postgres; GRANT ALL ON SCHEMA public TO PUBLIC; SET SESSION AUTHORIZATION 'edd'; SET search_path = public, pg_catalog; -- -- TOC entry 4 (OID 20101) -- Name: beancounter; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE beancounter ( "version" character varying(12) NOT NULL, data_last_updated timestamp with time zone ); -- -- TOC entry 5 (OID 20104) -- Name: stockinfo; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE stockinfo ( symbol character varying(12) DEFAULT ''::character varying NOT NULL, name character varying(64) DEFAULT ''::character varying NOT NULL, exchange character varying(16) DEFAULT ''::character varying NOT NULL, capitalisation real, low_52weeks real, high_52weeks real, earnings real, dividend real, p_e_ratio real, avg_volume integer, active boolean DEFAULT true ); -- -- TOC entry 6 (OID 20111) -- Name: stockprices; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE stockprices ( symbol character varying(12) DEFAULT ''::character varying NOT NULL, date date, previous_close real, day_open real, day_low real, day_high real, day_close real, day_change real, bid real, ask real, volume integer ); -- -- TOC entry 7 (OID 20115) -- Name: fxprices; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE fxprices ( currency character varying(12) DEFAULT ''::character varying NOT NULL, date date, previous_close real, day_open real, day_low real, day_high real, day_close real, day_change real ); -- -- TOC entry 8 (OID 20119) -- Name: portfolio; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE portfolio ( symbol character varying(16) DEFAULT ''::character varying NOT NULL, shares real, currency character varying(12), "type" character varying(16), "owner" character varying(16), cost real, date date ); -- -- TOC entry 9 (OID 20123) -- Name: cash; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE cash ( name character varying(16) DEFAULT ''::character varying NOT NULL, value double precision, currency character varying(12), "type" character varying(12), "owner" character varying(16), cost double precision, date date ); -- -- TOC entry 10 (OID 20127) -- Name: indices; Type: TABLE; Schema: public; Owner: edd -- CREATE TABLE indices ( symbol character varying(12) DEFAULT ''::character varying NOT NULL, stockindex character varying(12) DEFAULT ''::character varying NOT NULL ); -- -- TOC entry 11 (OID 20110) -- Name: stockinfo_pkey; Type: INDEX; Schema: public; Owner: edd -- CREATE UNIQUE INDEX stockinfo_pkey ON stockinfo USING btree (symbol); -- -- TOC entry 12 (OID 20114) -- Name: stockprices_pkey; Type: INDEX; Schema: public; Owner: edd -- CREATE UNIQUE INDEX stockprices_pkey ON stockprices USING btree (symbol, date); -- -- TOC entry 13 (OID 20118) -- Name: fxprices_pkey; Type: INDEX; Schema: public; Owner: edd -- CREATE UNIQUE INDEX fxprices_pkey ON fxprices USING btree (currency, date); -- -- TOC entry 14 (OID 20122) -- Name: portfolio_pkey; Type: INDEX; Schema: public; Owner: edd -- CREATE UNIQUE INDEX portfolio_pkey ON portfolio USING btree (symbol, "owner", date); -- -- TOC entry 15 (OID 20126) -- Name: cash_pkey; Type: INDEX; Schema: public; Owner: edd -- CREATE UNIQUE INDEX cash_pkey ON cash USING btree (name, "type", "owner", date); SET SESSION AUTHORIZATION 'postgres'; -- -- TOC entry 2 (OID 2200) -- Name: SCHEMA public; Type: COMMENT; Schema: -; Owner: postgres -- COMMENT ON SCHEMA public IS 'Standard public namespace'; beancounter-0.8.10/README0000644000000000000000000001554307361450616011673 0ustar BeanCounter -- A stock portfolio performance monitoring tool 1. Introduction Ever wondered what happened to your portfolio on a day the market moved 500 points? Ever wondered what your portfolio returned over the last (odd) period? Ever wondered if there was a simple cron job to report portfolio changes on a daily basis? Ever wondered if you could database the (public) price, volume, ... info on dozens of stocks for further analysis? BeanCounter does all this, and provides a convenient command-line tool as well as a Perl library that can be used by other applications. Beancounter support stocks from exchanges in the US, Canada, Britain, France, Germany, Italy, Singapore, Taiwan, Thailand, HongKong... as well as Australia and New Zealand. Tested patches for other markets are always welcome. US mutual funds are supported, as are foreign exchange rates and some precious metals. This is still somewhat beta in the sense that the command-line options and function interfaces might change. The (initial) release number is kept low intenionally. However, similar code has been working here since the fall of 1998. 2. How to get started You need Perl and PostgreSQL, or MySQL (as of version 0.4.0). Plus a host of Perl modules such as DBI (Database independent interface) with either the DBD ODBC or the DBD Postgres driver (i.e. DBD-Pg) or the DBD Mysql driver, LWP networking as well as Date::Manip and Statistics::Descriptive. All of this is readily available for Debian; or else in source code at www.postgresql.org and www.cpan.org. Upon opening the .tar.gz archive, run $ perl Makefile.PL $ make install and everything should be fine. Run $ beancounter --help for a quick check. Run 'setup_beancounter' to create the new database and tables, and to have them filled with example data. This even runs a first portfolio report on the (example) portfolio. The 'setup_beancounter' script has a few sanity checks to ensure that Postgres (or MySQL) is running, that the current user is a valid database user, that it is not called by root, and that the database hasn't already been created. 3. How to use it Beancounter's --help options shows the main functionality. This currently comprised the following operations: o "addindex index sym1 sym2 ..." fills the indices table with info on stocks comprising a market index. o "addportfolio sym1:nb1:fx sym2:nb:fx ..." fills the portfolio table with the specified quantities (eg 'nb1' or 'nb2' in the example above) of the specified stocks. Additional attributes of the holding can be specified as well. These are 'type' in which one could store the tax classification of a retirement account, eg 'rrsp' in Canada or '401k' in the USA, 'holder' which can carry the name of the account holder so that one could e.g. differentiate between spousal accounts, 'cost' which stores the purchase price and 'purchasedate' for the time of the purchase. o "addstock sym1 sym2 ..." to add one (or several) stock(s) to the database by filling both the 'stockinfo' table with general company and stock info, as well as the `stockprices' table with the most recent price data. Its purpose is to initialise the database for new stocks. o "backpopulate sym1 sym2 ..." backpopulates the database for one (or several) stock(s). It can be used to extend the database with historic prices. Limiting dates can be supplied via the --fromdate and --todate arguments. o "dailyjob" which combines 'update' and 'dayendreport' o "dayendreport" runs a report summarising day-over-day changes o "destroydb" removes the database entirely with not further warning. o "plreport" runs an end-of-day profit/loss report on the portfolio. It can be run automatically by cron(8) via an entry in crontab(5) such as 15 17 * * 1-5 beancounter update && beancounter plreport It can be run from cron(8) just after 'daily_update.pl'. o "risk" runs an risk analysis with parametric Value-at-Risk (VaR) calculated via the standard Delta method (a.k.a. RiskMetric (TM)), a non-parametric VaR (using the 1% quantile) as well as marginal VaR analysis. o "quote sym1 sym2 ..." retrieves price info from the Web and displays it without touching the database. This is useful for verifying a stock symbol as well the network status. o "status" computes a holdings value and (annualized) return summary which also shows how long a stock has been held. o "update" updates the database with day-end information. It can be run automatically by cron(8) via an entry in crontab(5) such as 15 17 * * 1-5 beancounter update (but also see about combining this with a call to beacounter plreport') o "warranty" shows a brief statement about the GNU General Public License The shell script 'setup_beancounter' initialises the database, creates the tables and fills them with initial data. The shell script 'update_beancounter' changes, if necessary, the database table to bring an older (0.1.*) installation up to speed. 5. To Do More error checking. More performance analytics. More markets. More documentation. Maybe a GUI. See the TODO file. 6. Disclaimer (taken straight from the GPL) NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. In short: if it breaks, you can keep the pieces. This is Free Software, use it as you see fit, but don't come running or screaming if it doesn't work as intended. Dirk Eddelbuettel $Id: README,v 1.6 2001/10/12 01:47:53 edd Exp edd $ beancounter-0.8.10/contrib/0000755000000000000000000000000010423306424012431 5ustar beancounter-0.8.10/contrib/getDiv0000644000000000000000000000624110102066601013573 0ustar #!/usr/bin/perl -w # # contributed to beancounter by # Joao Antunes Costa new; $tree->parse($content); my $txt = HTML::FormatText->new; my @text = reverse(split("\n", $txt->format($tree))); my $work = 0; my $IsSplit = 0; my $lastDate; my $firstDate = "1980-01-01"; my $ratio = 0; my $lastRatio = 0; my $sql; my ($one, $two); foreach (@text) { chomp($_); $work = 0 if (/Adj Close/i); if ($work) { s/^ +| +$//; if (/^(\d[^:]+):([^s]+)/i) { $IsSplit = 1; $one = $1; $two = $2; $one =~ s/^\s+|\s+$//; $two =~ s/^\s+|\s+$//; } if (/^\d.+\d$/ && $IsSplit) { $IsSplit = 0; $lastDate = FormatDate($_); $ratio = $two / $one; if (DateOk($firstDate, $lastDate) || $lastRatio != $ratio) { $lastRatio = $ratio; #print "$ratio at $lastDate\n"; $sql = "UPDATE stockprices SET day_open = day_open * $ratio, day_low = day_low * $ratio, day_high = day_high * $ratio, day_close = day_close * $ratio, volume = volume * $ratio WHERE symbol = '$symbol' AND date < '$lastDate';\n"; print $sql; $firstDate = $lastDate; } else { warn "Ignoring $two/$one split at $lastDate for $symbol\n"; } } } $work = 1 if (/Close price/i); } } else { die "could not get $symbol\n"; } sub DateOk { my ($firstDate, $lastDate) = @_; #print "checking $firstDate vs $lastDate\n"; my @d1 = split "-", $firstDate; my @d2 = split "-", $lastDate; my $days = abs(Delta_Days ($d1[0], $d1[1], $d1[2], $d2[0], $d2[1], $d2[2])); return 1 if ($days > 40); return 0; } sub FormatDate { my $date = shift; my @date = split("-", $date); #print "Formatting $date\n"; if ($date[2] > 85) {$date[2] = "19".$date[2];} else {$date[2] = "20".$date[2];} return $date[2] . "-" . GetMonth($date[1]) . "-" . stuff($date[0]); } sub stuff { my $day = shift; return "0$day" if (length($day) == 1); return $day; } sub GetMonth { my $month = shift; if ( $month =~ /Jan/i) { return "01"; } elsif ( $month =~ /Feb/i) { return "02"; } elsif ( $month =~ /Mar/i) { return "03"; } elsif ( $month =~ /Apr/i) { return "04"; } elsif ( $month =~ /May/i) { return "05"; } elsif ( $month =~ /Jun/i) { return "06"; } elsif ( $month =~ /Jul/i) { return "07"; } elsif ( $month =~ /Aug/i) { return "08"; } elsif ( $month =~ /Sep/i) { return "09"; } elsif ( $month =~ /Oct/i) { return "10"; } elsif ( $month =~ /Nov/i) { return "11"; } elsif ( $month =~ /Dec/i) { return "12"; } else { die("Unrecognized month string ($month)"); } } beancounter-0.8.10/contrib/schnapp0000644000000000000000000001734210203033724014013 0ustar #!/usr/bin/perl -w # vim: set sw=4 si et: # # schnapp - show the best dividend yield of your stocks # # Copyright 2002-2005 by Mathias Weidner # # Licensed under GPL2, look at end of file for details. # # Use 'perldoc schnapp' to view the manpage # or 'pod2man schnapp > schnapp.1' to create a manpage file for *roff # use strict; $ENV{PATH} = join ":", qw(/bin /usr/bin); $|++; use Getopt::Long; use Finance::BeanCounter; # global variables: my $RCS_VERSION = '$Id: schnapp,v 1.3 2005/02/10 21:26:27 mathias Exp $'; my $db_min_schema = "0.6.0"; # minimum version of the database that we need use vars qw($help $index $debug $verbose $version $fxarg $datearg $prevdatearg $rcfile $extrafx $updatedate $type $max $dbsystem $dbname $fxupdate); # my $rcfile = $ENV{HOME} . "/.beancounterrc"; ($prevdatearg, $datearg, $fxupdate, $max) = ("6 month ago", "today", 1, 10); my %options = ( "help" => \$help, "index=s" => \$index, "max=i" => \$max, "type=s" => \$type, "version" => \$version, ); GetOptions(%options) || die "ERROR: No such option. -help for help\n"; &help if ($help); &version if ($version); my $command = shift @ARGV; my %Config = GetConfig( $rcfile, $debug, $verbose, $fxarg, $extrafx, $updatedate, $dbsystem, $dbname, $fxupdate, $command ); my $dbh = ConnectToDb(); if (TestInsufficientDatabaseSchema($dbh, $db_min_schema)) { warn "Database schema is not current. Please run 'update_beancounter'\n"; } else { schnapp(); } CloseDB($dbh); # #-=-=-=-=-=-=-=-=-=-=-=-=-=-=- sub schnapp { my $stocks = select_stocks(); my $portfl = select_portfolio(); my $count = 1; printf("Nr. %-10s %-10s %-20s %-6s %-6s %-5s %-5s %-6s\n", "Date", "Symbol", "Name", "Low", "High", "Div", "Divr", "Shares"); my $sort = sub { return -1 if (not defined $stocks->{$b}->{divr}); return 1 if (not defined $stocks->{$a}->{divr}); return ($stocks->{$b}->{divr} <=> $stocks->{$a}->{divr}); }; foreach my $sym (sort $sort keys %{$stocks}) { if ($count <= $max or $portfl->{$sym}) { printf( "%3i %-10s %-10s %-20s %6.2f %6.2f %5.2f %5.2f %6i\n", $count, $stocks->{$sym}->{date}, $sym, $stocks->{$sym}->{name}, $stocks->{$sym}->{day_low}, $stocks->{$sym}->{day_high}, $stocks->{$sym}->{div}, $stocks->{$sym}->{divr}, $portfl->{$sym} ? $portfl->{$sym}->{shares} : 0 ); } $count++; } } # schnapp() sub select_stocks { my $sqli = <do('create temporary table tmp (symbol varchar(12),date date)'); $dbh->do( 'insert into tmp select symbol, max(date) from stockprices group by symbol' ); my $sth; if ($index) { $sth = $dbh->prepare($sqli); $sth->execute($index); } else { $sth = $dbh->prepare($sql); $sth->execute(); } my $stocks = $sth->fetchall_hashref('symbol'); $dbh->do('drop table tmp'); return $stocks; } # select_stocks() sub select_portfolio { my $sql = <prepare($sqlt); $sth->execute($type); } else { $sth = $dbh->prepare($sql); $sth->execute(); } my $portfolio = $sth->fetchall_hashref('symbol'); return $portfolio; } # select_portfolio() sub version { print < and B and finally shows you a list of stocks ordered by descending dividend yield. The I is determined by dividing the C from table B by C from table B and multiplying the result with 100 for each stock. =head1 OPTIONS =over 4 =item B<--help> A short message listing the purpose and the options of the program. =item B<--index>=I You may limit the stocks which will be investigated to a certain stock index that must be known in beancounter (see C in the L manual). =item B<--max>=I You may limit the count of stocks that will be displayed. The program lists I stocks with descending dividend yield and thereafter only those stocks that are in the portfolio. =item B<--type>=I You may limit the stocks of your portfolio which will be displayed to I. This relates to the type in the beancounter portfolio. See the L manpage. =item B<--version> Shows the version and exits. =back =head1 ENVIRONMENT =over 4 =item C<$HOME> The value of the environment variable C<$HOME> is used to determine the location of the B configuration file. =item C<$PATH> The environment variable C<$PATH> is set to C. =back =head1 FILES =head2 B<$HOME/.beancounterrc> The configuration file for B is used to get access to the database. =head1 NOTES This program uses the database filled by your daily run of C but takes only the last dates of each stock. It determines automatically the last date when beancounter updated the stockprices. =head1 SEE ALSO L =head1 AUTHOR Copyright 2002-2005 by Mathias Weidner =head1 COPYRIGHT AND LICENSE This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. =head1 HISTORY 2005-02-10 mw Copyright, License, Manpage, Option '--version' 2002 mw Initial Version beancounter-0.8.10/contrib/beancounter.spec0000644000000000000000000000434410423306331015614 0ustar %define perl_vendorlib %(eval "`perl -V:installvendorlib`"; echo $installvendorlib) %define perl_vendorarch %(eval "`perl -V:installvendorarch`"; echo $installvendorarch) Summary: BeanCounter portfolio performance toolkit Name: beancounter Version: 0.8.6 Release: 1 License: GNU GPL Group: Applications/Finance BuildRoot: /var/tmp/build-rpm URL: http://dirk.eddelbuettel.com/code/beancounter.html Source: http://dirk.eddelbuettel.com/code/beancounter/beancounter-%{version}.tar.gz Requires: perl-Statistics-Descriptive %description Ever wondered what happened to your portfolio on a day the market moved 500 points? Ever wondered what your portfolio returned over the last (odd and arbitrary) period? Ever wondered what the Value-at-Risk (VaR) was? Ever wondererd what the marginal risk contribution of a given stock in your portfolio was? Ever wondered if you could easily database the (public) prices info for further analysis? Ever wondered if there was a simple cron job to report all this on a daily basis? BeanCounter does all this, and provides an easy-to-use command-line tool as well as a Perl module that can be used with other pursuits. It stores its data (price, volume, earnings --- whatever Yahoo! supplies) in a PostgreSQL relational database system. BeanCounter works with equities from exchanges in the US, Canada, Europe and Asia. Options, foreign exchange rates, some commodities as well as US mutual funds are also supported as the data is provided by Yahoo! %changelog %prep %setup -q rm -rf $RPM_BUILD_ROOT mkdir -p $RPM_BUILD_ROOT %build # perl Makefile.PL CFLAGS="%{optflags}" %{__perl} Makefile.PL \ PREFIX="%{buildroot}%{_prefix}" \ INSTALLDIRS="vendor" # make %{__make} %{?_smp_mflags} %install make PREFIX=$RPM_BUILD_ROOT/usr SITEPREFIX=$RPM_BUILD_ROOT/usr install %clean rm -rf $RPM_BUILD_ROOT %files %defattr(-,root,root) %doc README THANKS TODO example.beancounterrc beancounter.html %doc *txt contrib/* /usr/bin/* %doc %{_mandir}/man*/* %{perl_vendorlib}/* %changelog * Tue Dec 28 2004 R P Herrold 0.7.6-1orc - rework a .spec file dateing from 0.4.0 to a proper one, which will build as non-root, and properly cascade version information with a minimal edit beancounter-0.8.10/BeanCounter.pm0000644000000000000000000021132011405253765013545 0ustar # # BeanCounter.pm --- A stock portfolio performance monitoring toolkit # # Copyright (C) 1998 - 2010 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # $Id: BeanCounter.pm,v 1.107 2010/06/13 22:13:09 edd Exp $ package Finance::BeanCounter; require strict; require Exporter; #use Carp; # die with info on caller use Data::Dumper; # debugging aid use Date::Manip; # for date parsing use DBI; # for the Perl interface to the database use English; # friendlier variable names use Finance::YahooQuote; # fetch quotes from Yahoo! use POSIX qw(strftime); # for date formatting use Statistics::Descriptive; # simple statistical functions use Text::ParseWords; # parse .csv data more reliably @ISA = qw(Exporter); # make these symbols known @EXPORT = qw(BeanCounterVersion CloseDB ConnectToDb TestInsufficientDatabaseSchema DatabaseDailyData DatabaseHistoricalData DatabaseHistoricalFXData DatabaseHistoricalUBCFX DatabaseHistoricalOandAFX DatabaseInfoData ExistsDailyData ExistsFXDailyData GetTodaysAndPreviousDates GetCashData GetConfig GetDate GetDailyData GetFXData GetFXDatum GetOandAFXData GetUBCFXData GetUBCFXHash GetYahooCurrency GetIsoCurrency GetHistoricalData GetPortfolioData GetPriceData GetRetracementData GetRiskData ParseDailyData ParseNumeric PrintHistoricalData ReportDailyData Sign UpdateDatabase UpdateFXDatabase UpdateFXviaUBC UpdateTimestamp ); @EXPORT_OK = qw( ); %EXPORT_TAGS = (all => [@EXPORT_OK]); my $VERSION = sprintf("%d.%d", q$Revision: 1.107 $ =~ /(\d+)\.(\d+)/); my %Config; # local copy of configuration hash sub BeanCounterVersion { return $VERSION; } sub ConnectToDb { # log us into the database (PostgreSQL) my $hoststr = ''; $hoststr = "host=$Config{host}" unless (grep(/^$Config{host}$/, ('localhost','127.0.0.1','::1/128'))); my $dsn = 'dbi:'; if ($Config{odbc}) { $dsn .= "ODBC:$Config{dsn}"; } elsif (lc $Config{dbsystem} eq "postgresql") { $dsn .= "Pg:dbname=$Config{dbname};${hoststr}"; } elsif (lc $Config{dbsystem} eq "mysql") { $dsn .= "mysql:dbname=$Config{dbname};${hoststr}"; } elsif (lc $Config{dbsystem} eq "sqlite") { $dsn .= "SQLite:dbname=$Config{dbname}"; $Config{user} = ''; $Config{passwd} = ''; } elsif (lc $Config{dbsystem} eq "sqlite2") { $dsn .= "SQLite2:dbname=$Config{dbname}"; $Config{user} = ''; $Config{passwd} = ''; } else { die "Database system $Config{dbsystem} is not supported\n"; } my $dbh = DBI->connect($dsn, $Config{user}, $Config{passwd}, { PrintError => $Config{debug}, Warn => $Config{verbose}, AutoCommit => 0 }); die "No luck with database connection\n" unless ($dbh); return $dbh; } sub CloseDB { my $dbh = shift; $dbh->disconnect or warn $dbh->errstr; } sub ConvertVersionToLargeInteger($) { my ($txt) = @_; my ($major,$minor,$revision) = ($txt =~ m/^([0-9]+)\.([0-9]+)\.([0-9]+)$/); my $numeric = $major * 1e6 + $minor * 1e3 + $revision; #print "[$txt] -> [$major] [$minor] [$revision] -> $numeric\n"; return($numeric); } sub TestInsufficientDatabaseSchema($$) { my ($dbh, $required) = @_; my @tables = $dbh->tables(); die "Database does not contain table beancounter. " . "Please run 'update_beancounter'.\n" unless grep /beancounter/, @tables; my $sql = q{select version from beancounter}; my @res = $dbh->selectrow_array($sql) or die $dbh->errstr; my $dbschema = $res[0]; my $num_required = ConvertVersionToLargeInteger($required); my $num_schema = ConvertVersionToLargeInteger($dbschema); print "Database has schema $dbschema, we require version $required\n" if $Config{debug}; return ($num_schema < $num_required); # extensive testing was required =:-) } sub GetTodaysAndPreviousDates { my ($date, $prev_date); my $today = DateCalc(ParseDate("today"), "- 8 hours"); # Depending on whether today is a working day, use today # or the most recent preceding working day if (Date_IsWorkDay($today)) { $date = UnixDate($today, "%Y%m%d"); $prev_date = UnixDate(DateCalc($today, "- 1 business days"), "%Y%m%d"); } else { $date = UnixDate(DateCalc($today, "- 1 business days"), "%Y%m%d"); $prev_date = UnixDate(DateCalc($today, "- 2 business days"), "%Y%m%d"); } # override with optional dates, if supplied $date = UnixDate(ParseDate($main::datearg), "%Y%m%d") if ($main::datearg); $prev_date = UnixDate(ParseDate($main::prevdatearg),"%Y%m%d") if ($main::prevdatearg); # and create 'prettier' non-ISO 8601 form my $pretty_date = UnixDate(ParseDate($date), "%d %b %Y"); my $pretty_prev_date = UnixDate(ParseDate($prev_date), "%d %b %Y"); return ($date, $prev_date, $pretty_date, $pretty_prev_date); } sub GetConfig { my ($file, $debug, $verbose, $fx, $extrafx, $updatedate, $dbsystem, $dbname, $fxupdate, $commit, $equityupdate, $ubcfx, $hostarg, $command) = @_; %Config = (); # reset hash $Config{debug} = $debug; # no debugging as default $Config{verbose} = $verbose; # silent == non-verbose as default $Config{odbc} = 0; # if 1, use DBI-ODBC, else use DBI-Pg $Config{currency} = "USD"; # default to US dollars as domestic currency $Config{user} = $ENV{USER}; # default user is current user $Config{passwd} = undef; # default password is no password $Config{dbsystem} = "PostgreSQL"; $Config{dbname} = "beancounter"; $Config{today} = strftime("%Y%m%d", localtime); ($Config{lastbizday}, $Config{prevbizday}) = GetTodaysAndPreviousDates; # DSN name for ODBC $Config{dsn} = "beancounter"; # default ODBC data source name # default to updating FX if ($fxupdate) { $Config{fxupdate} = 1; } else { $Config{fxupdate} = 0; } # default to committing to db if ($commit) { $Config{commit} = 1; } else { $Config{commit} = 0; } # default to updateing stocks too if ($equityupdate) { $Config{equityupdate} = 1; } else { $Config{equityupdate} = 0; } # default to updateing stocks too if ($ubcfx) { $Config{ubcfx} = 1; } else { $Config{ubcfx} = 0; } # pre-load a default host argument $Config{host} = $hostarg if defined($hostarg); unless ( -f $file ) { warn "Config file $file not found, ignored.\n"; } else { open (FILE, "<$file") or die "Cannot open $file: $!\n"; while () { next if (m/(\#|%)/); # ignore comments, if any next if (m/^\s*$/); # ignore empty lines, if any if (m/^\s*(\w+)\s*=\s*(.+)\s*$/) { $Config{$1} = "$2"; } } close(FILE); } $Config{currency} = $fx if defined($fx); $Config{dbname} = $dbname if defined($dbname); $Config{dbsystem} = $dbsystem if defined($dbsystem); $Config{odbc} = 1 if defined($dbsystem) and lc $dbsystem eq "odbc"; # but allow command-line argument to override $Config{host} = $hostarg if defined($hostarg) and $hostarg ne "localhost"; if (defined($extrafx)) { unless ($command =~ /^(update|dailyjob)$/) { warn "Warning: --extrafx ignored as not updating db\n"; } else { $Config{extrafx} = $extrafx if defined($extrafx); } } if (defined($updatedate)) { # test the updatedate argument unless ($command =~ /^(update|dailyjob)$/) { warn "Warning: --updatedate ignored as not updating db\n"; } else { die "Error: Invalid date $updatedate for --forceupdate\n" unless (ParseDate($updatedate)); $Config{updatedate} = UnixDate(ParseDate($updatedate),"%Y%m%d"); } } print Dumper(\%Config) if $Config{debug}; return %Config; } sub GetCashData { my ($dbh, $date, $res) = @_; my ($stmt, $sth, $rv, $ary_ref, $sym_ref, %cash); my ($name, $value, $fx, $cost); # get the symbols $stmt = "select name, value, currency, cost from cash "; $stmt .= "where value > 0 "; $stmt .= "and $res " if ( defined($res) and $res =~ m/(name|value|currency|cost|owner)/i and not $res =~ m/(symbol|shares|exchange|day)/i ); $stmt .= "order by name"; print "GetCashData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); $rv = $sth->execute(); # run query for report end date while (($name, $value, $fx, $cost) = $sth->fetchrow_array) { $cash{$name}{value} += $value; # adds if there are several $cash{$name}{fx} = $fx; $cash{$name}{cost} = $cost; } $sth->finish(); return(\%cash); } sub GetDailyData { # use Finance::YahooQuote::getquote # This uses the 'return an entire array' approach of Finance::YahooQuote. my @Args = @_; if (defined($Config{proxy})) { $Finance::YahooQuote::PROXY = $Config{proxy}; } if (defined($Config{firewall}) and $Config{firewall} ne "" and $Config{firewall} =~ m/.*:.*/) { my @q = split(':', $Config{firewall}, 2); $Finance::YahooQuote::PROXYUSER = $q[0]; $Finance::YahooQuote::PROXYPASSWD = $q[1]; } if (defined($Config{timeout})) { $Finance::YahooQuote::TIMEOUT = $Config{timeout} if $Config{timeout}; } #my $url = "http://quote.yahoo.com/d" . # "?f=snl1d1t1c1p2va2bapomwerr1dyj1x&s="; #my $array = GetQuote($url,@NA); # get all North American quotes my $array = getquote(@Args); # get North American quotes my @Res; push @Res, (@$array); # and store the entire array of arrays print Dumper(\@Res) if $Config{debug}; return @Res; } ## Simple routine to get quotes for an array of arguments BEGIN { use HTTP::Request::Common; } sub GetUBCFXData { my ($symbolsref, $from, $to) = @_; my @symbols = @$symbolsref; my $nsym = $#symbols + 1; my $base = $Config{currency}; # instead of unconditionally requesting USD ## we need the dates as yyyy, mm and dd my ($fy,$fm,$fd,$ty,$tm,$td); ($fy,$fm,$fd) = ($from =~ m/(\d\d\d\d)(\d\d)(\d\d)/); ($ty,$tm,$td) = ($to =~ m/(\d\d\d\d)(\d\d)(\d\d)/); ## build the query URL my $url = "http://fx.sauder.ubc.ca/cgi/fxdata?b=$base&"; $url .= "ld=$td&lm=$tm&ly=$ty&fd=$fd&fm=$fm&fy=$fy&"; $url .= "daily&q=volume&f=csv&o=T.C"; $url .= "&c=" . join("&c=", @symbols); print "Url is $url\n" if $Config{debug}; my @qr; # results will be collected here my $ua = RequestAgent->new; $ua->env_proxy; # proxy settings from *_proxy env. variables. $ua->proxy('http', $PROXY) if defined $PROXY; $ua->timeout($TIMEOUT) if defined $TIMEOUT; foreach (split('\015?\012',$ua->request(GET $url)->content)) { ## skip the commercials / copyrights / attributions next if $_ =~ m/(PACIFIC|Prof\. Werner Antweiler)/; print "--> $_\n" if $Config{debug}; ## split the csv stream with quotewords() from Text::ParseWords my @q = quotewords(',', 0, $_); my @fx = splice(@q, -$nsym); # last $nsym are the quotes push (@qr, [$q[1], @fx]); print $q[1], " ", join(" ", @fx), "\n" if $Config{debug}; } return \@qr; } ## wrapper for single-day hash of currencies sub GetUBCFXHash { my ($symref, $date) = @_; my $res = GetUBCFXData($symref, $date, $date); my @symbols = @$symref; my $nsym = $#symbols + 1; ## format is like ## YYYY/MM/DD CAD/USD GBP/USD ## 2005/01/31 1.2380 0.53087 ## so loop over all columns but first my %res; for (my $i=0; $i<$nsym; $i++) { ## the currency comes as, e.g., CAD/USD so split the CAD part of my $cur = (split(/\//, $res->[0]->[$i+1]))[0]; print $cur, "\t" , $res->[1]->[$i+1], "\n" if $Config{debug}; ## and value is matching entry in second row $res{$cur} = $res->[1]->[$i+1]; } return \%res; # return the new hash } ## get FX data from OandA.com sub GetOandAFXData { my ($symbol, $from, $to) = @_; my $base = $Config{currency}; # instead of unconditionally requesting USD ## we need the dates as yyyy, mm and dd my ($fy,$fm,$fd,$ty,$tm,$td); ($fy,$fm,$fd) = ($from =~ m/(\d\d\d\d)(\d\d)(\d\d)/); ($ty,$tm,$td) = ($to =~ m/(\d\d\d\d)(\d\d)(\d\d)/); ## build the query URL my $url = "http://www.oanda.com/convert/fxhistory?lang=en&"; $url .= "date1=$fm%2F$fd%2F$fy&"; $url .= "date=$tm%2F$td%2F$ty&date_fmt=us&"; $url .= "exch=$symbol&exch2=&expr=$Config{currency}&expr2="; $url .= "&margin_fixed=0&SUBMIT=Get+Table&format=CSV&redirected=1"; print "Url is $url\n" if $Config{debug}; my @qr; # results will be collected here my $ua = RequestAgent->new; $ua->env_proxy; # proxy settings from *_proxy env. variables. $ua->proxy('http', $PROXY) if defined $PROXY; $ua->timeout($TIMEOUT) if defined $TIMEOUT; my $state = 0; foreach (split('\015?\012',$ua->request(GET $url)->content)) { my $line = $_; if ($state == 0) { if ($_ =~ m|
|) {
	$state += 1;
	$line =~ s|
||;
      }	    
      #next;
    }
    if ($state == 1) {
      $state += 1 if $_ =~ m|
|;
      #next;
    }
    next unless $state == 1;
    #print "--> $_\n" if $Config{debug};
    #$state = $_ !~ m|
|; ## split the csv stream with quotewords() from Text::ParseWords #my @q = quotewords(',', 0, $_); #my @fx = splice(@q, -$nsym); # last $nsym are the quotes #push (@qr, [$q[1], @fx]); #print $q[1], " ", join(" ", @fx), "\n" if $Config{debug}; push (@qr, $line); print $line, "\n" if $Config{debug}; } return \@qr; } sub getIso2YahooCurrencyHashRef() { # map between ISO country codes and Yahoo symbols for the Philly exchange return {"AUD" => "^XAY", # was "^XAD", "AUDUSD=X", "CAD" => "^XCV", # was "^XCD", "CADUSD=X", "CHF" => "^XSY", # was "^XSF", "CHFUSD=X", "EUR" => "^XEU", # was "EURUSD=X", "GBP" => "^XBX", # was "^XBP", "GBPUSD=X", "JPY" => "^XJZ", # was "^XJY", "JPYUSD=X", "USD" => "----"}; } sub GetYahooCurrency($) { my ($isoCurrency) = @_; my $ref = getIso2YahooCurrencyHashRef(); return $ref->{$isoCurrency}; } sub GetIsoCurrency($) { my ($yahooCurrency) = @_; my $ref = getIso2YahooCurrencyHashRef(); # Reverse the hash table, ie. yahoo => iso: my %yahoo2isoHash = map { $ref->{$_} => $_ } keys(%$ref); return $yahoo2isoHash{$yahooCurrency}; } sub GetHistoricalData { # get a batch of historical quotes from Yahoo! my ($symbol,$from,$to) = @_; my $ua = RequestAgent->new; $ua->env_proxy; # proxy settings from *_proxy env. variables. $ua->proxy('http', $Config{proxy}) if $Config{proxy}; # or config vars my ($a,$b,$c,$d,$e,$f); # we need the date as yyyy, mm and dd ($c,$a,$b) = ($from =~ m/(\d\d\d\d)(\d\d)(\d\d)/); ($f,$d,$e) = ($to =~ m/(\d\d\d\d)(\d\d)(\d\d)/); --$a; --$d; # month is zero-based my $req = new HTTP::Request GET => "http://table.finance.yahoo.com/" . "table.csv?a=$a&b=$b&c=$c&d=$d&e=$e&f=$f&s=$symbol&y=0&g=d&ignore=.csv"; my $res = $ua->request($req); # Pass request to user agent and get response if ($res->is_success) { # Check the outcome of the response return split(/\n/, $res->content); } else { warn "No luck with symbol $symbol\n"; } } sub GetPortfolioData { my ($dbh, $res) = @_; my ($stmt, $sth); # get the portfolio data $stmt = "select symbol, shares, currency, type, owner, cost, date "; $stmt .= "from portfolio "; $stmt .= "where $res" if (defined($res)); print "GetPortfolioData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); $sth->execute(); my $data_ref = $sth->fetchall_arrayref({}); return $data_ref; } sub GetPriceData { my ($dbh, $date, $res) = @_; my ($stmt, $sth, $rv, $ary_ref, @symbols, %dates); my ($ra, $symbol, $name, $shares, $currency, $price, $prevprice, %prices, %prev_prices, %shares, %fx, %name, %purchdate, %cost, $cost,$pdate,%pricedate); # get the symbols $stmt = "select distinct p.symbol from portfolio p, stockinfo s "; $stmt .= "where s.symbol = p.symbol and s.active "; $stmt .= qq{and p.symbol in (select distinct symbol from portfolio where $res) } if (defined($res)); $stmt .= "order by p.symbol"; print "GetPriceData():\n\$stmt = $stmt\n" if $Config{debug}; # get symbols @symbols = @{ $dbh->selectcol_arrayref($stmt) }; # for each symbol, get most recent date subject to supplied date $stmt = qq{select max(date) from stockprices where symbol = ? and day_close > 0 and date <= ? }; print "GetPriceData():\n\$stmt = $stmt\n" if $Config{debug}; # for each symbol, get most recent date subject to supplied date:\n"; foreach $ra (@symbols) { if (!defined($sth)) { $sth = $dbh->prepare($stmt); } $rv = $sth->execute($ra, $date); # run query for report end date my $res = $sth->fetchrow_array; $dates{$ra} = $res; $sth->finish() if $Config{odbc}; } #sum(p.shares*p.cost)/sum(p.shares) as p.cost, # now get closing price etc at date $stmt = qq{select i.symbol, i.name, p.shares, p.currency, d.day_close, p.cost, p.date, d.previous_close from stockinfo i, portfolio p, stockprices d where d.symbol = p.symbol and i.symbol = d.symbol and d.date = ? and d.symbol = ? }; #### TWA, 2003-12-04 ## According to the original code, here the restriction applies to the ## portfolio table only. But _note_: ## the same restriction is used in GetRiskData() !!!! ## the same restriction is used in GetRetracementData() !!!! ## But it is not enough to restrict the symbols used by the sub-select ## command. One has to restrict the main selection with the same ## restriction rules. ## Thus, make a copy of the restriction and replace the column names ## to a syntax to use the portfolio table only. if (defined($res)) { ## avoid name space pollution my $portfolio_restriction = $res; $portfolio_restriction =~ s/\bsymbol\b/p\.symbol/g; $portfolio_restriction =~ s/\bshares\b/p\.shares/g; $portfolio_restriction =~ s/\bcurrency\b/p\.currency/g; $portfolio_restriction =~ s/\btype\b/p\.type/g; $portfolio_restriction =~ s/\bowner\b/p\.owner/g; $portfolio_restriction =~ s/\bcost\b/p\.cost/g; $portfolio_restriction =~ s/\bdate\b/p\.date/g; $stmt .= qq{ and $portfolio_restriction } } # end if (defined($res)) $stmt .= qq{ and d.symbol in (select distinct symbol from portfolio where $res) } if (defined($res)); ## $stmt .= qq{ group by i.symbol,i.name,p.shares,p.currency,d.day_close,p.date,d.previous_close }; #select symbol, avg('today'-date) as days, sum(shares*cost)/sum(shares) as cost, sum(shares) as size, sum(shares*cost) as pos from portfolio where owner!='peter' group by symbol order by days desc; print "GetPriceData():\n\$stmt = $stmt\n" if $Config{debug}; # now get closing price etc at date $sth = undef; my $i = 0; foreach $ra (@symbols) { if (!defined($sth)) { $sth = $dbh->prepare($stmt); } $rv = $sth->execute($dates{$ra}, $ra); while (($symbol, $name, $shares, $currency, $price, $cost, $pdate, $prevprice) = $sth->fetchrow_array) { print join " ", ($symbol, $name, $shares, $currency, $price, $cost||"NA", $pdate||"NA", $prevprice||"NA"), "\n" if $Config{debug}; $fx{$name} = $currency; $prices{$name} = $price; $pricedate{$name} = $dates{$symbol}; $cost{$name} = $cost; $purchdate{$name} = $pdate; $prev_prices{$name} = $prevprice; $name .= ":$i"; $i++; $shares{$name} = $shares; $purchdate{$name} = $pdate; # also store purchuse date on non-aggregate entry $cost{$name} = $cost; # also store purchuse cost on non-aggregate entry } $sth->finish; } print Dumper(\%prices) if $Config{debug}; print Dumper(\%prev_prices) if $Config{debug}; print Dumper(\%shares) if $Config{debug}; return (\%fx, \%prices, \%prev_prices, \%shares, \%pricedate, \%cost, \%purchdate); } sub GetFXData { my ($dbh, $date, $fx) = @_; ## find FX data from closest date smaller or equal to the requested date # for each symbol, get most recent date subject to supplied date my $stmt = qq{select max(date) from fxprices where currency = ? and date <= ? }; print "GetFXData():\n\$stmt = $stmt\n" if $Config{debug}; # get most recent date subject to supplied date my %fxdates; my $sth; foreach my $fxval (sort values %$fx) { next if $fxval eq $Config{currency};# skip user's default currency if (!defined($sth)) { $sth = $dbh->prepare($stmt); } $rv = $sth->execute($fxval, $date); # run query for report end date my $res = $sth->fetchrow_array; $fxdates{$fxval} = $res; $sth->finish() if $Config{odbc}; } $stmt = qq{ select day_close, previous_close from fxprices where date = ? and currency = ? }; print "GetFXData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = undef; my (%fx_prices,%prev_fx_prices); foreach my $fxval (sort values %$fx) { if ($fxval eq $Config{currency}) { $fx_prices{$fxval} = 1.0; $prev_fx_prices{$fxval} = 1.0; } else { if (!defined($sth)) { $sth = $dbh->prepare($stmt); } $sth->execute($fxdates{$fxval}, $fxval); # run query for FX cross my ($val, $prevval) = $sth->fetchrow_array or die "Found no $fxval for $date in the beancounter database.\n " . "Use the --date and/or --prevdate options to pick another date.\n"; $fx_prices{$fxval} = $val; $prev_fx_prices{$fxval} = $prevval; if (Date_Cmp(ParseDate($fxdates{$fxval}), ParseDate($date)) !=0) { print "Used FX date $fxdates{$fxval} instead of $date\n" if $Config{verbose}; } my $ary_ref = $sth->fetchall_arrayref; } } return (\%fx_prices, \%prev_fx_prices); } ## simple wrapper for GetFXDate for single currency + date sub GetFXDatum { my ($dbh, $date, $fx) = @_; my %fxhash; $fxhash{foo} = $fx; my ($fxcurrent) = GetFXData($dbh, $date, \%fxhash); return $fxcurrent->{$fx}; } ## NB no longer used as we employ Finance::YahooQuote directly sub GetQuote { # taken from Dj's Finance::YahooQuote my ($URL,@symbols) = @_; # and modified to allow for different URL my ($x,@q,@qr,$ua,$url); # and the simple filtering below as well # the firewall code below if (defined($Config{proxy})) { $Finance::YahooQuote::PROXY = $Config{proxy}; } if (defined($Config{firewall}) and $Config{firewall} ne "" and $Config{firewall} =~ m/.*:.*/) { my @q = split(':', $Config{firewall}, 2); $Finance::YahooQuote::PROXYUSER = $q[0]; $Finance::YahooQuote::PROXYPASSWD = $q[1]; } if (defined($Config{timeout})) { $Finance::YahooQuote::TIMEOUT = $Config{timeout} if $Config{timeout}; } undef @qr; # reset result structure while (scalar(@symbols) > 0) {# while we have symbols to query my (@symbols_100); # Peter Kim's patch to batch 100 at a time if (scalar(@symbols)>=100) {# if more than hundred symbols left @symbols_100 = splice(@symbols,0,100); # then skim the first 100 off } else { # otherwise @symbols_100 = @symbols; # take what's left @symbols = (); # and show we're done } my $array = getquote(@symbols_100); # get quotes using Finance::YahooQ. push(@qr,[@array]); # and store result as anon array } return \@qr; # return a pointer to the results array } sub GetRetracementData { my ($dbh,$date,$prevdate,$res,$fx_prices) = @_; my (%high52, %highprev, %low52, %lowprev); # get the symbols my $stmt = qq{select distinct p.symbol, i.name, p.shares, p.date from portfolio p, stockinfo i where p.symbol = i.symbol and i.active }; #### TWA, 2003-12-07 ## According to the original code, here the restriction applies to the ## portfolio table only. But _note_: ## the same restriction is used in GetPriceData() !!!! ## But it is not enough to restrict the symbols used by the sub-select ## command. One has to restrict the main selection with the same ## restriction rules. ## Thus, make a copy of the restriction and replace the column names ## to a syntax to use the portfolio table only. if (defined($res)) { ## avoid name space pollution my $portfolio_restriction = $res; $portfolio_restriction =~ s/\bsymbol\b/p\.symbol/g; $portfolio_restriction =~ s/\bshares\b/p\.shares/g; $portfolio_restriction =~ s/\bcurrency\b/p\.currency/g; $portfolio_restriction =~ s/\btype\b/p\.type/g; $portfolio_restriction =~ s/\bowner\b/p\.owner/g; $portfolio_restriction =~ s/\bcost\b/p\.cost/g; $portfolio_restriction =~ s/\bdate\b/p\.date/g; $stmt .= qq{ and $portfolio_restriction } } # end if (defined($res)) $stmt .= qq{and p.symbol in (select distinct symbol from portfolio where $res) } if (defined($res)); $stmt .= "order by p.symbol"; print "GetRetracementData():\n\$stmt = $stmt\n" if $Config{debug}; my $sth = $dbh->prepare($stmt); my $rv = $sth->execute(); # run query for report end date my $sref = $sth->fetchall_arrayref; # # get static 52max from stockinfo # $stmt = qq{select high_52weeks, low_52weeks # from stockinfo where symbol = ?}; # $sth = $dbh->prepare($stmt); # foreach my $ra (@$sref) { # $rv = $sth->execute($ra->[0]); # my @res = $sth->fetchrow_array; # get data # $high52{$ra->[1]} = $res[0]; # $low52{$ra->[1]} = $res[1]; # } # get max/min over prevate .. date period $stmt = qq{select day_close from stockprices where symbol = ? and date <= ? and date >= ? and day_close > 0 order by date }; print "GetRetracementData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); foreach my $ra (@$sref) { my $refdate = $prevdate; # start from previous date if (defined($ra->[3])) { # if startdate in DB ## then use it is later then the $prevdate $refdate = $ra->[3] if (Date_Cmp($prevdate, $ra->[3]) < 0) } $rv = $sth->execute($ra->[0], $date, $refdate); my $dref = $sth->fetchall_arrayref; # get data my $x = Statistics::Descriptive::Full->new(); for (my $i=0; $iadd_data($dref->[$i][0]); # add prices } $highprev{$ra->[1]} = $x->max(); $lowprev{$ra->[1]} = $x->min(); } # return (\%high52, \%highprev, \%low52, \%lowprev); return (\%highprev, \%lowprev); } sub GetRiskData { my ($dbh,$date,$prevdate,$res,$fx_prices,$crit) = @_; # get the symbols my $stmt = qq{select distinct p.symbol, i.name from portfolio p, stockinfo i where p.symbol = i.symbol and i.active }; #### TWA, 2003-12-07 ## According to the original code, here the restriction applies to the ## portfolio table only. But _note_: ## the same restriction is used in GetPriceData() !!!! ## But it is not enough to restrict the symbols used by the sub-select ## command. One has to restrict the main selection with the same ## restriction rules. ## Thus, make a copy of the restriction and replace the column names ## to a syntax to use the portfolio table only. if (defined($res)) { ## avoid name space pollution my $portfolio_restriction = $res; $portfolio_restriction =~ s/\bsymbol\b/p\.symbol/g; $portfolio_restriction =~ s/\bshares\b/p\.shares/g; $portfolio_restriction =~ s/\bcurrency\b/p\.currency/g; $portfolio_restriction =~ s/\btype\b/p\.type/g; $portfolio_restriction =~ s/\bowner\b/p\.owner/g; $portfolio_restriction =~ s/\bcost\b/p\.cost/g; $portfolio_restriction =~ s/\bdate\b/p\.date/g; $stmt .= qq{ and $portfolio_restriction } } # end if (defined($res)) $stmt .= qq{and p.symbol in (select distinct symbol from portfolio where $res) } if (defined($res)); $stmt .= "order by p.symbol"; print "GetRiskData():\n\$stmt = $stmt\n" if $Config{debug}; my $sth = $dbh->prepare($stmt); my $rv = $sth->execute(); # run query for report end date my $sref = $sth->fetchall_arrayref; # compute volatility $stmt = qq{select day_close from stockprices where symbol = ? and date <= ? and date >= ? and day_close > 0 order by date }; print "GetRiskData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); my (%vol, %quintile); foreach my $ra (@$sref) { $rv = $sth->execute($ra->[0], $date, $prevdate); my $dref = $sth->fetchall_arrayref; # get data my $x = Statistics::Descriptive::Full->new(); for (my $i=1; $iadd_data($dref->[$i][0]/$dref->[$i-1][0] - 1); } printf("%16s: stdev %6.2f min %6.2f max %6.2f\n", $ra->[1], $x->standard_deviation, $x->min, $x->max) if $Config{debug}; $vol{$ra->[1]} = $x->standard_deviation; if ($x->count() < 100) { print "$ra->[1]: Only ", $x->count(), " data points, ", "need at least 100 for percentile calculation\n" if $Config{debug}; $quintile{$ra->[1]} = undef; } else { $quintile{$ra->[1]} = $x->percentile(1); } } # compute correlations via OLS regression $stmt = qq{select a.day_close, b.day_close from stockprices a, stockprices b where a.symbol = ? and b.symbol = ? and a.date <= ? and a.date >= ? and a.date = b.date and a.day_close != 0 and b.day_close != 0 order by a.date }; print "GetRiskData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); my %cor; foreach my $ra (@$sref) { foreach my $rb (@$sref) { my $res = $ra->[0] cmp $rb->[0]; if ($res < 0) { $rv = $sth->execute($ra->[0], $rb->[0], $date, $prevdate); my $dref = $sth->fetchall_arrayref; # get data my $x = Statistics::Descriptive::Full->new(); my $y = Statistics::Descriptive::Full->new(); for (my $i=1; $iadd_data($dref->[$i][0]/$dref->[$i-1][0] - 1); $y->add_data($dref->[$i][1]/$dref->[$i-1][1] - 1); } my @arr = $x->least_squares_fit($y->get_data()); my $rho = $arr[2]; unless (defined($rho)) { warn "No computable correlation between $ra->[1] and $rb->[1];" . " set to 0\n"; $rho = 0.0; } $cor{$ra->[1]}{$rb->[1]} = $rho; printf("%6s %6s correlation %6.4f\n", $ra->[1], $rb->[1], $arr[2]) if $Config{debug}; } elsif ($res > 0) { $cor{$ra->[1]}{$rb->[1]} = $cor{$rb->[1]}{$ra->[1]}; } else { $cor{$ra->[1]}{$rb->[1]} = 1; } } } # for each symbol, get most recent date subject to supplied date my %maxdate; $stmt = qq{select max(date) from stockprices where symbol = ? and date <= ? }; print "GetRiskData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); foreach my $ra (@$sref) { $rv = $sth->execute($ra->[0], $date); # run query for report end date my $res = $sth->fetchrow_array; $maxdate{$ra->[1]} = $res; $sth->finish() if $Config{odbc}; } # get position values my (%pos, $possum); $stmt = qq{select p.shares, d.day_close, p.currency from portfolio p, stockprices d, stockinfo i where d.symbol = p.symbol and d.symbol = i.symbol and d.date = ? and d.symbol = ? }; $stmt .= qq{and d.symbol in (select distinct symbol from portfolio where $res) } if (defined($res)); print "GetRiskData():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); foreach my $ra (@$sref) { $rv = $sth->execute($maxdate{$ra->[1]}, $ra->[0]); while (my ($shares, $price, $fx) = $sth->fetchrow_array) { print "$ra->[1] $shares $price\n" if $Config{debug}; my $amount = $shares * $price * $fx_prices->{$fx} / $fx_prices->{$Config{currency}}; $pos{$ra->[1]} += $amount; } } # aggregate risk: # VaR is z_crit * sqrt(horizon) * sqrt (X.transpose * Sigma * X) # where X is position value vector and Sigma the covariance matrix # given that Perl is not exactly a language for matrix calculus (as # eg GNU Octave), we flatten the computation into a double loop my $sum = 0; foreach my $pkey (keys %pos) { if (defined($pos{$pkey}) && defined($vol{$pkey})) { foreach my $vkey (keys %vol) { if (defined($pos{$vkey}) && defined($vol{$vkey}) && defined($cor{$vkey}{$pkey})) { $sum += $pos{$pkey} * $pos{$vkey} * $vol{$vkey} * $vol{$pkey} * $cor{$vkey}{$pkey}; } } } } my $var = $crit * sqrt($sum); ## marginal var my %margvar; foreach my $outer (keys %pos) { my $saved = $pos{$outer}; my $sum = 0; $pos{$outer} = 0; foreach my $pkey (keys %pos) { if (defined($pos{$pkey}) && defined($vol{$pkey})) { foreach my $vkey (keys %vol) { if (defined($pos{$vkey}) && defined($vol{$vkey}) && defined($cor{$vkey}{$pkey})) { $sum += $pos{$pkey} * $pos{$vkey} * $vol{$vkey} * $vol{$pkey} * $cor{$vkey}{$pkey}; } } } } $margvar{$outer} = $crit * sqrt($sum) - $var; $pos{$outer} = $saved; } return ($var, \%pos, \%vol, \%quintile, \%margvar); } sub DatabaseDailyData { # a row to the dailydata table my ($dbh, %hash) = @_; my @cols = ('previous_close', 'day_open', 'day_high', 'day_low', 'day_close', 'day_change', 'bid', 'ask', 'volume'); my @updTerms = (); foreach my $col (@cols) { push(@updTerms, "$col = ?"); } my $updStmt = 'update stockprices set ' . join(', ', @updTerms) . ' where symbol = ? and date = ?'; print "$updStmt\n" if $Config{debug}; my $updSth; push(@cols, 'symbol', 'date'); my @insTerms = (); foreach my $col (@cols) { push(@insTerms, '?'); } my $insStmt = 'insert into stockprices (' . join(', ', @cols) . ') values (' . join(', ', @insTerms) . ')'; print "$insStmt\n" if $Config{debug}; my $insSth; foreach my $key (keys %hash) { # now split these into reference to the arrays print "$hash{$key}{symbol} " if $Config{verbose}; if ($hash{$key}{date} eq "N/A") { warn "Not databasing $hash{$key}{symbol}\n" if $Config{debug}; next; } if (ExistsDailyData($dbh, %{$hash{$key}})) { my @vals = (); foreach my $col (@cols) { if ($hash{$key}{$col} =~ m/^\s*N\/A\s*$/) { push(@vals, undef); } else { push(@vals, $hash{$key}{$col}); } } if ($Config{commit}) { if (!defined($updSth)) { $updSth = $dbh->prepare($updStmt) or die $dbh->errstr; } $updSth->execute(@vals) and $updSth->finish() or warn $dbh->errstr . "Update failed for " . "$hash{$key}{symbol} with [$updStmt]\n"; } } else { my @vals = (); foreach my $col (@cols) { if ($hash{$key}{$col} =~ m/^\s*N\/A\s*$/) { push(@vals, undef); } else { push(@vals, $hash{$key}{$col}); } } if ($Config{commit}) { if (!defined($insSth)) { $insSth = $dbh->prepare($insStmt) or die $dbh->errstr; } $insSth->execute(@vals) and $insSth->finish() or warn $dbh->errstr . "Insert failed for " . "$hash{$key}{symbol} with [$insStmt]\n"; } } } $dbh->commit() if $Config{commit}; } sub DatabaseFXDailyData { my ($dbh, %hash) = @_; foreach my $key (keys %hash) { # now split these into reference to the arrays if ($key eq "") { print "Empty key in DatabaseFXDailyData, skipping\n" if $Config{debug}; next; } my $fx = GetIsoCurrency($hash{$key}{symbol}); print "$fx ($hash{$key}{symbol}) " if $Config{debug}; if (ExistsFXDailyData($dbh, $fx, %{$hash{$key}})) { # different sequence of parameters, see SQL statement above! my $stmt = qq{update fxprices set previous_close = ?, day_open = ?, day_low = ?, day_high = ?, day_close = ?, day_change = ? where currency = ? and date = ? }; print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; print "DatabaseFXDailyData(): $hash{$key}{previous_close}, $hash{$key}{day_open}, $hash{$key}{day_low}, $hash{$key}{day_high}, $hash{$key}{day_close}, $hash{$key}{day_change}, $fx, $hash{$key}{date} \n" if $Config{debug}; if ($Config{commit}) { $dbh->do($stmt, undef, $hash{$key}{previous_close}, $hash{$key}{day_open}, $hash{$key}{day_low}, $hash{$key}{day_high}, $hash{$key}{day_close}, $hash{$key}{day_change}, $fx, $hash{$key}{date} ) or warn "Failed for $fx at $hash{$key}{date}\n"; } ## Alternate FX using the EURUSD=X quotes which don;t have history # my $stmt = qq{update fxprices # set day_close = ? # where currency = ? # and date = ? # }; # print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; # print "DatabaseFXDailyData(): ", # "$hash{$key}{day_close}, $fx, $hash{$key}{date} \n" if $Config{debug}; # if ($Config{commit}) { # $dbh->do($stmt, undef, # $hash{$key}{day_close}, # $fx, # $hash{$key}{date} # ) # or warn "Failed for $fx at $hash{$key}{date}\n"; # } } else { my $stmt = qq{insert into fxprices values (?, ?, ?, ?, ?, ?, ?, ?);}; print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; print "DatabaseFXDailyData(): $fx, $hash{$key}{date}, $hash{$key}{previous_close}, $hash{$key}{day_open}, $hash{$key}{day_low}, $hash{$key}{day_high}, $hash{$key}{day_close}, $hash{$key}{day_change}, \n" if $Config{debug}; if ($Config{commit}) { my $sth = $dbh->prepare($stmt); $sth->execute($fx, $hash{$key}{date}, $hash{$key}{previous_close}, $hash{$key}{day_open}, $hash{$key}{day_low}, $hash{$key}{day_high}, $hash{$key}{day_close}, $hash{$key}{day_change} ) or warn "Failed for $fx at $hash{$key}{date}\n"; } ## Alternate FX using the EURUSD=X quotes which don;t have history # my $stmt = qq{insert into fxprices values (?, ?, ?, ?, ?, ?, ?, ?);}; # print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; # print "DatabaseFXDailyData(): $fx, $hash{$key}{date},", # "$hash{$key}{day_close}\n" if $Config{debug}; # if ($Config{commit}) { # my $sth = $dbh->prepare($stmt); # $sth->execute($fx, $hash{$key}{date}, # undef, undef, undef, undef, # $hash{$key}{day_close}, undef # ) # or warn "Failed for $fx at $hash{$key}{date}\n"; # } } if ($Config{commit}) { $dbh->commit(); } } } sub DatabaseHistoricalData { my ($dbh, $symbol, @res) = @_; $symbol = uc $symbol; # make sure symbols are uppercase'd my %data = (symbol => $symbol, date => undef, day_open => undef, day_high => undef, day_low => undef, day_close => undef, volume => undef); my @colNames = sort(keys(%data)); my @colRepl = (); my @updTerms = (); foreach my $col (@colNames) { push(@colRepl, '?'); next if ($col eq 'symbol' || $col eq 'date'); push(@updTerms, "$col = ?"); } my $insStmt = 'insert into stockprices (' . join(', ', @colNames) . ') values (' . join(', ', @colRepl) . ')'; my $insSth; my $updStmt = 'update stockprices set ' . join(', ', @updTerms) . ' where symbol = ? and date = ?'; my $updSth; print "DatabaseHistoricalData: insStmt is \"$insStmt\"\n" if $Config{debug}; print "DatabaseHistoricalData: updStmt is \"$updStmt\"\n" if $Config{debug}; foreach my $line (@res) { # loop over all supplied symbols next if !defined($line); ($data{date}, $data{day_open}, $data{day_high}, $data{day_low}, $data{day_close}, $data{volume}, $data{adjclose}) = split(/\,/, $line); $data{date} = GetDate($data{date}); if (defined($data{date})) { # If close was not supplied, we assume a mutual fund. # So let close be open. if (!defined($data{day_close})) { $data{day_close} = $data{day_open}; $data{day_open} = undef; } elsif (defined($data{adjclose}) && $data{adjclose} != $data{day_close} && $data{day_close} != 0) { # process split adjustment factor my $split_adj = $data{adjclose} / $data{day_close}; $data{day_open} *= $split_adj; $data{day_high} *= $split_adj; $data{day_low} *= $split_adj; $data{day_close} = $data{adjclose}; } if (ExistsDailyData($dbh, %data)) { my @colVals = (); foreach my $col (@colNames) { next if ($col eq 'symbol' || $col eq 'date'); $data{$col} = 'NULL' if !defined($data{$col}); push(@colVals, $data{$col}); } push(@colVals, $data{symbol}, $data{date}); if (!defined($updSth)) { $updSth = $dbh->prepare($updStmt) or die $dbh->errstr; } $updSth->execute(@colVals) or die $updSth->errstr; $updSth->finish(); } else { my @colVals = (); foreach my $col (@colNames) { $data{$col} = 'NULL' if !defined($data{$col}); push(@colVals, $data{$col}); } if (!defined($insSth)) { $insSth = $dbh->prepare($insStmt) or die $dbh->errstr; } $insSth->execute(@colVals) or die $insSth->errstr; $insSth->finish(); } } } $dbh->commit() if $Config{commit}; print "Done with $symbol\n" if $Config{verbose}; } sub DatabaseHistoricalFXData { my ($dbh, $symbol, @res) = @_; my $checked = 0; # flag to ensure not nonsensical or errors my %data; # hash to store data of various completenesses my $cut = UnixDate(ParseDate("30-Dec-2003"), "%Y%m%d"); my $fx = GetIsoCurrency($symbol); foreach $ARG (@res) { # loop over all supplied symbols next if m/^<\!-- .*-->/; # skip lines with html comments (April 2004) # make sure the first line of data is correct so we don't insert garbage if ($checked==0 and m/Date(,Open,High,Low)?,Close(,Volume)?/) { $checked = tr/,//; print "Checked now $checked\n" if $Config{verbose}; } elsif ($checked) { my ($date, $open, $high, $low, $close, $volume, $cmd); # based on the number of elements, ie columns, we split the parsing if ($checked eq 5 or $checked eq 6) { ($date, $open, $high, $low, $close, $volume) = split(/\,/, $ARG); $date = UnixDate(ParseDate($date), "%Y%m%d"); %data = (symbol => $fx, date => $date, day_open => $open, day_high => $high, day_low => $low, day_close => $close, volume => undef); # never any volume info for FX } else { # no volume for indices print "Unknown currency format: $ARG\n"; } if (Date_Cmp($date,$cut) >= 0) { # if date if on or after cutoff date $data{day_open} /= 100.0; # then scale by a hundred to match the $data{day_low} /= 100.0; # old level "in dollars" rather than the $data{day_high} /= 100.0; # new one "in cents" $data{day_close} /= 100.0; } # now given the data, decide whether we add new data or update old data if (ExistsFXDailyData($dbh,$fx,%data)) { # update data if it exists $cmd = "update fxprices set "; ##$cmd .= "volume = $data{volume}," if defined($data{volume}); $cmd .= "day_open = $data{day_open}," if defined($data{day_open}); $cmd .= "day_low = $data{day_low}," if defined($data{day_low}); $cmd .= "day_high = $data{day_high}," if defined($data{day_high}); $cmd .= "day_close = $data{day_close} " . "where currency = '$data{symbol}' " . "and date = '$data{date}'"; } else { # insert $cmd = "insert into fxprices (currency, date,"; $cmd .= "day_open," if defined($data{day_open}); $cmd .= "day_high," if defined($data{day_high}); $cmd .= "day_low," if defined($data{day_low}); $cmd .= "day_close"; ##$cmd .= ",volume" if defined($data{volume}); $cmd .= ") values ('$data{symbol}', '$data{date}', "; $cmd .= "$data{day_open}," if defined($data{day_open}); $cmd .= "$data{day_high}," if defined($data{day_high}); $cmd .= "$data{day_low}," if defined($data{day_low}); $cmd .= "$data{day_close}"; ##$cmd .= ",$data{volume} " if defined($data{volume}); $cmd .= ");"; } if ($Config{commit}) { print "$cmd\n" if $Config{debug}; $dbh->do($cmd) or die $dbh->errstr; $dbh->commit(); } } else { ; # do nothing with bad data } } print "Done with $fx (using $symbol)\n" if $Config{verbose}; } sub DatabaseHistoricalUBCFX { my ($dbh, $aref, @arg) = @_; my ($cmd, %data); foreach my $lref (@$aref) { # loop over all retrieved data next if $lref->[0] eq "YYYY/MM/DD"; $data{date} = UnixDate(ParseDate($lref->[0]), "%Y%m%d"); my $i = 1; foreach my $fx (@arg) { if (ExistsFXDailyData($dbh,$fx,%data)) { # update data if it exists $cmd = "update fxprices set "; $cmd .= "day_close = " . 1.0/$lref->[$i] . " " . "where currency = '$fx' and date = '$data{date}'"; } else { $cmd = "insert into fxprices (currency, date, day_close) "; $cmd .= "values ('$fx', '$data{date}', 1.0/$lref->[$i] )"; } $i++; if ($Config{commit}) { print "$cmd\n" if $Config{debug}; $dbh->do($cmd) or die $dbh->errstr; } } #print "Done with $fx (using $symbol)\n" if $Config{verbose}; } if ($Config{commit}) { $dbh->commit(); } } sub DatabaseHistoricalOandAFX { my ($dbh, $aref, @arg) = @_; my ($cmd, %data); foreach my $line (@$aref) { # loop over all retrieved data ## split the csv stream with quotewords() from Text::ParseWords my @q = quotewords(',', 0, $line); $data{date} = UnixDate(ParseDate($q[0]), "%Y%m%d"); my $i = 1; foreach my $fx (@arg) { if (ExistsFXDailyData($dbh,$fx,%data)) { # update data if it exists $cmd = "update fxprices set "; $cmd .= "day_close = " . $q[1] . " " . "where currency = '$fx' and date = '$data{date}'"; } else { $cmd = "insert into fxprices (currency, date, day_close) "; $cmd .= "values ('$fx', '$data{date}', $q[1] )"; } $i++; if ($Config{commit}) { print "$cmd\n" if $Config{debug}; $dbh->do($cmd) or die $dbh->errstr; } } #print "Done with $fx (using $symbol)\n" if $Config{verbose}; } if ($Config{commit}) { $dbh->commit(); } } sub DatabaseInfoData { # initialise a row in the info table my ($dbh, %hash) = @_; foreach my $key (keys %hash) { # now split these into reference to the arrays # check stockinfo for $key if ( ExistsInfoSymbol($dbh, %{$hash{$key}}) ) { warn "DatabaseInfoData(): Symbol $key already in stockinfo table\n" if ( $Config{verbose} ); next; } my $cmd = "insert into stockinfo (symbol, name, exchange, " . " capitalisation, low_52weeks, high_52weeks, earnings, " . " dividend, p_e_ratio, avg_volume, active) " . "values('$hash{$key}{symbol}'," . $dbh->quote($hash{$key}{name}) . ", " . " '$hash{$key}{exchange}', " . " $hash{$key}{market_capitalisation}," . " $hash{$key}{'52_week_low'}," . " $hash{$key}{'52_week_high'}," . " $hash{$key}{earnings_per_share}," . " $hash{$key}{dividend_per_share}," . " $hash{$key}{price_earnings_ratio}," . " $hash{$key}{average_volume}," . " '1')"; $cmd =~ s|'?N/A'?|null|g; # convert (textual) "N/A" into (database) null print "$cmd\n" if $Config{debug}; print "$hash{$key}{symbol} " if $Config{verbose}; if ($Config{commit}) { $dbh->do($cmd) or die $dbh->errstr; $dbh->commit(); } } } sub ExistsInfoSymbol { my ($dbh, %hash) = @_; if (!defined($_symExistsInfoSymbolSth)) { $_symExistsInfoSymbolSth = $dbh->prepare(qq{select symbol from stockinfo where symbol = ?}) or die $dbh->errstr; } $_symExistsInfoSymbolSth->execute($hash{symbol}) or die $_symExistsInfoSymbolSth->errstr; my @rows = $_symExistsInfoSymbolSth->fetchrow_array(); $_symExistsInfoSymbolSth->finish(); # plausibility tests here # someone might care to extend this to consider the 'active' tuple # maybe if it's false that fact should be noted since # the user has apparently seen fit to add it to the database (again) return (@rows > 0); } sub ExistsDailyData($%) { my ($dbh, %hash) = @_; if (!defined($_symExistsDailyDataSth)) { $_symExistsDailyDataSth = $dbh->prepare(qq{select symbol from stockprices where symbol = ? and date = ?}) or die $dbh->errstr; } $_symExistsDailyDataSth->execute($hash{symbol}, $hash{date}) or die $_symExistsDailyDataSth->errstr; my @rows = $_symExistsDailyDataSth->fetchrow_array(); $_symExistsDailyDataSth->finish(); return (@rows > 0); } sub ExistsFXDailyData { my ($dbh,$fx,%hash) = @_; my $stmt = qq{select previous_close, day_open, day_low, day_high, day_close, day_change from fxprices where currency = ? and date = ? }; print "ExistsFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; my $sth = $dbh->prepare($stmt); $sth->execute($fx,$hash{date}); my @row = $sth->fetchrow_array(); $sth->finish(); return (@row > 0); } sub GetDate { # date can be "4:01PM" (same day) or "Jan 15" my ($value) = @_; # Date::Manip knows how to deal with them... return UnixDate(ParseDate($value), "%Y%m%d"); } sub ParseDailyData { # stuff the output into the hash my @rra = @_; # we receive an array with references to arrays my %hash; # we return a hash of hashes foreach my $ra (@rra) { # now split these into reference to the arrays my $key = $ra->[0]; $hash{$key}{symbol} = uc $ra->[0]; $hash{$key}{name} = RemoveTrailingSpace($ra->[1]); $hash{$key}{day_close} = ParseNumeric($ra->[2]); unless ($hash{$key}{date} = GetDate($ra->[3])) { $hash{$key}{date} = "N/A"; warn "Ignoring symbol $key with unparseable date\n"; } $hash{$key}{time} = $ra->[4]; $hash{$key}{day_change} = ParseNumeric($ra->[5]); $hash{$key}{percent_change} = $ra->[6]; $hash{$key}{volume} = $ra->[7]; $hash{$key}{average_volume} = $ra->[8]; $hash{$key}{bid} = ParseNumeric($ra->[9]); $hash{$key}{ask} = ParseNumeric($ra->[10]); $hash{$key}{previous_close} = ParseNumeric($ra->[11]); $hash{$key}{day_open} = ParseNumeric($ra->[12]); my (@tmp) = split / - /, $ra->[13]; $hash{$key}{day_low} = ParseNumeric($tmp[0]); $hash{$key}{day_high} = ParseNumeric($tmp[1]); (@tmp) = split / - /, $ra->[14]; $hash{$key}{'52_week_low'} = ParseNumeric($tmp[0]); $hash{$key}{'52_week_high'} = ParseNumeric($tmp[1]); $hash{$key}{earnings_per_share} = $ra->[15]; $hash{$key}{price_earnings_ratio} = $ra->[16]; $hash{$key}{dividend_date} = $ra->[17]; $hash{$key}{dividend_per_share} = $ra->[18]; $hash{$key}{yield} = $ra->[19]; if ($ra->[20] =~ m/(\S*)B$/) { # convert to millions from billions $hash{$key}{market_capitalisation} = $1*(1e3); } elsif ($ra->[20] =~ m/(\S*)T$/) { # reported in trillions -- convert to millions $hash{$key}{market_capitalisation} = $1*(1e6); } elsif ($ra->[20] =~ m/(\S*)M$/) { # keep it in millions $hash{$key}{market_capitalisation} = $1; } elsif ($ra->[20] =~ m/(\S*)K$/) { # reported in thousands -- convert to millions $hash{$key}{market_capitalisation} = $1*(1e-3); } else { # it's not likely a number at all -- pass it on $hash{$key}{market_capitalisation} = $ra->[20]; } $hash{$key}{exchange} = RemoveTrailingSpace($ra->[21]); } return %hash } sub ParseNumeric { # parse numeric fields which could be fractions my $v = shift; # expect one argument $v =~ s/\s*$//; # kill trailing whitespace $v =~ s/\+//; # kill leading plus sign if ($v =~ m|(.*) (.*)/(.*)|) {# if it is a fraction return $1 + $2/$3; # return the decimal value } else { # else return $v; # return the value itself } } sub PrintHistoricalData { # simple display routine for hist. data my (@res) = @_; my $i=1; foreach $ARG (@res) { next if m/^<\!-- .*-->/; # skip lines with html comments (April 2004) print $i++, ": $ARG\n"; } } sub RemoveTrailingSpace { my $txt = shift; $txt =~ s/\s*$//; return $txt; } sub ReportDailyData { # detailed display / debugging routine my (%hash) = @_; foreach my $key (keys %hash) { # now split these into reference to the arrays printf "Name %25s\n", $hash{$key}{name}; printf "Symbol %25s\n", $hash{$key}{symbol}; printf "Exchange %25s\n", $hash{$key}{exchange}; printf "Date %25s\n", $hash{$key}{date}; printf "Time %25s\n", $hash{$key}{time}; printf "Previous Close %25s\n", $hash{$key}{previous_close}; printf "Open %25s\n", $hash{$key}{day_open}; printf "Day low %25s\n", $hash{$key}{day_low}; printf "Day high %25s\n", $hash{$key}{day_high}; printf "Close %25s\n", $hash{$key}{day_close}; printf "Change %25s\n", $hash{$key}{day_change}; printf "Percent Change %25s\n", $hash{$key}{percent_change}; printf "Bid %25s\n", $hash{$key}{bid}; printf "Ask %25s\n", $hash{$key}{ask}; printf "52-week low %25s\n", $hash{$key}{'52_week_low'}; printf "52-week high %25s\n", $hash{$key}{'52_week_high'}; printf "Volume %25s\n", $hash{$key}{volume}; printf "Average Volume %25s\n", $hash{$key}{average_volume}; printf "Dividend date %25s\n", $hash{$key}{dividend_date}; printf "Dividend / share %25s\n", $hash{$key}{dividend_per_share}; printf "Dividend yield %25s\n", $hash{$key}{yield}; printf "Earnings_per_share %25s\n", $hash{$key}{earnings_per_share}; printf "P/E ratio %25s\n", $hash{$key}{price_earnings_ratio}; printf "Market Capital %25s\n", $hash{$key}{market_capitalisation}; } } sub ScrubDailyData { # stuff the output into the hash my %hash = @_; # we receive ## Check the date supplied from Yahoo! ## ## The first approach was to count all dates for a given market ## This works well when you have, say, 3 Amex and 5 NYSE stock, and ## Yahoo just gets one date wrong -- we can then compare the one "off-date" ## against, say, four "good" dates and override ## Unfortunately, this doesn't work so well for currencies where you ## typically only get one, or maybe two, and have nothing to compare against ## ## my %date; # date comparison hash ## foreach my $key (keys %hash) {# store all dates for market ## $date{$hash{$key}{exchange}}{$hash{$key}{date}}++; # and count'em ## } ## -- and later ## if ($date{$hash{$key}{exchange}}{$hash{$key}{date}} # and outnumbered ## < $date{$hash{$key}{exchange}}{$Config{today}}) { ## warn("Override: $hash{$key}{name}: $hash{$key}{date} has only " . ## "$date{$hash{$key}{exchange}}{$hash{$key}{date}} votes,\n\tbut " . ## "$hash{$key}{exchange} has " . ## "$date{$hash{$key}{exchange}}{$Config{today}} " . ## "votes for $Config{today}"); ## $hash{$key}{date} = $Config{today}; ## } else { ## warn("$hash{$key}{name} has date $hash{$key}{date}, " . ## "not $Config{today} but no voting certainty"); ## } ## ## $date{$hash{$key}{exchange}}{$Config{today}} = 0 ## unless defined($date{$hash{$key}{exchange}}{$Config{today}}); ## ## So now we simply override if (and only if) the --forceupdate ## argument is used. This is still suboptimal if eg you are running this ## on public holidays. We will have to find a way to filter this ## foreach my $key (keys %hash) {# now check the date if ($hash{$key}{date} eq "N/A") { # if Yahoo! gave us no data if ($hash{$key}{symbol} =~ /^\^X/) { # and it was currency my $retry = GetIsoCurrency($hash{$key}{symbol}) . "USD=X"; my @retrysymbols; push @retrysymbols, $retry; my (@newarr) = GetDailyData(@retrysymbols); print "Retrying $retry:\n", Dumper(@newarr) if $Config{debug}; foreach my $ra (@newarr) { # split these into ref. to the arrays #print "$ra->[0]\n"; #$hash{$key}{symbol} = uc $ra->[0]; $hash{$key}{name} = RemoveTrailingSpace($ra->[1]); $hash{$key}{day_close} = ParseNumeric($ra->[2]); $hash{$key}{day_open} = $hash{$key}{day_low} = $hash{$key}{day_high} = $hash{$key}{previous_close} = $hash{$key}{day_change} = -1.2345; $hash{$key}{date} = GetDate($ra->[3]); $hash{$key}{time} = $ra->[4]; } } else { warn "Not scrubbing $hash{$key}{symbol}\n" if $Config{debug}; next; } } if ($hash{$key}{date} ne $Config{today}) { # if date is not today my $age = Delta_Format(DateCalc($hash{$key}{date}, $Config{lastbizday}, undef, 2), "approx", 0, "%dt"); if ($age > 5) { warn "Ignoring $hash{$key}{symbol} ($hash{$key}{name}) " . "with old date $hash{$key}{date}\n"; #warn "Ignoring $hash{$key}{name} with old date $hash{$key}{date}\n"; #if $Config{debug}; $hash{$key}{date} = "N/A"; next; } if (defined($Config{updatedate})) { # and if we have an override $hash{$key}{date} = $Config{updatedate}; # use it warn "Overriding date for $hash{$key}{symbol} ($hash{$key}{name}) " . "to $Config{updatedate}\n"; #warn "Overriding date for $hash{$key}{name} to $Config{updatedate}\n"; } else { warn "$hash{$key}{symbol} ($hash{$key}{name}) " . "has date $hash{$key}{date}\n"; #warn "$hash{$key}{name} has date $hash{$key}{date}\n"; } } if ($hash{$key}{previous_close} ne "N/A" and ($hash{$key}{day_close} == $hash{$key}{previous_close}) and ($hash{$key}{day_change} != 0)) { $hash{$key}{previous_close} = $hash{$key}{day_close} - $hash{$key}{day_change}; warn "Adjusting previous close for $key from close and change\n"; } # Yahoo! decided, on 2004-02-26, to change the ^X indices from # US Dollar to US Cent, apparently. if ($hash{$key}{symbol} =~ /^\^X/) { if (Date_Cmp(ParseDate($hash{$key}{date}), ParseDate("20040226")) > 0 and not Date_Cmp(ParseDate($hash{$key}{date}), ParseDate("20050117")) > 0) { warn "Scaling $key data from dollars to pennies\n" if $Config{debug}; $hash{$key}{previous_close} /= 100; $hash{$key}{day_open} /= 100; $hash{$key}{day_low} /= 100; $hash{$key}{day_high} /= 100; $hash{$key}{day_close} /= 100; $hash{$key}{day_change} /= 100; } } } return %hash; } sub Sign { my $x = shift; if ($x > 0) { return 1; } elsif ($x < 0){ return -1; } else { return 0; } } sub UpdateDatabase { # update content in the db at end of day my ($dbh, $res) = @_; my ($stmt, $sth, $rv, $ra, @symbols); $stmt = qq{ select distinct symbol from stockinfo where symbol != '' and active }; $stmt .= qq{ and symbol in (select distinct symbol from portfolio where $res) } if defined($res); $stmt .= " order by symbol;"; print "UpdateDatabase():\n\$stmt = $stmt\n" if $Config{debug}; @symbols = @{ $dbh->selectcol_arrayref($stmt) }; print join " ", @symbols, "\n" if $Config{verbose}; my @arr = GetDailyData(@symbols);# retrieve _all_ the data my %data = ParseDailyData(@arr); # put it into a hash %data = ScrubDailyData(%data); # and "clean" it ReportDailyData(%data) if $Config{verbose}; UpdateInfoData($dbh, %data); DatabaseDailyData($dbh, %data); UpdateTimestamp($dbh); } sub UpdateFXDatabase { my ($dbh, $res) = @_; # get all non-USD symbols (no USD as we don't need a USD/USD rate) my $stmt = qq{ select distinct currency from portfolio where symbol != '' and currency != 'USD' }; $stmt .= " and $res " if (defined($res)); print "UpdateFXDatabase():\n\$stmt = $stmt\n" if $Config{debug}; my @symbols = map { GetYahooCurrency($ARG) } @{ $dbh->selectcol_arrayref($stmt)}; print "UpdateFXDatabase(): Symbols are ", join(" ", @symbols), "\n" if $Config{debug}; if ($Config{extrafx}) { foreach my $arg (split /,/, $Config{extrafx}) { push @symbols, GetYahooCurrency($arg); } } if (scalar(@symbols) > 0) { # if there are FX symbols my @arr = GetDailyData(@symbols); # retrieve _all_ the data my %data = ParseDailyData(@arr); %data = ScrubDailyData(%data); # and "clean" it ReportDailyData(%data) if $Config{verbose}; DatabaseFXDailyData($dbh, %data); } UpdateTimestamp($dbh); } ## use alternate FX data supply from the PACIFIC / Sauder School / UBC sub UpdateFXviaUBC { my ($dbh, $res) = @_; # get all non-USD symbols (no USD as we don't need a USD/USD rate) my $stmt = qq{ select distinct currency from portfolio where symbol != '' and currency != 'USD' }; $stmt .= " and $res " if (defined($res)); print "UpdateFXviaUBC():\n\$stmt = $stmt\n" if $Config{debug}; my @symbols = @{ $dbh->selectcol_arrayref($stmt) }; print "UpdateFXviaUBC() -- symbols=" . join(" ", @symbols) . "\n" if $Config{debug}; my %data; $data{date} = $Config{lastbizday}; $data{date} = $Config{updatedate} if exists($Config{updatedate}); ## also fetch data via the PACIFIC server at Sauder / UBC my $ubcfx = GetUBCFXHash(\@symbols, $data{date}, $data{date}); print "UBC server results\n", Dumper($ubcfx) if $Config{debug}; foreach my $key (keys %{$ubcfx}) { # split these into reference to the arrays my $fx = $key; #$yahoo2iso->{$hash{$key}{symbol}}; print "Looking at $fx\n" if $Config{debug}; if (ExistsFXDailyData($dbh, $fx, %data)) { my $stmt = qq{update fxprices set day_close = ? where currency = ? and date = ? }; print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; print "DatabaseFXDailyData(): 1/$ubcfx->{$fx}, $fx, $data{date} \n" if $Config{debug}; if ($Config{commit}) { $dbh->do($stmt, undef, 1/$ubcfx->{$fx}, $fx, $data{date}) or warn "Failed for $fx at $data{date}\n"; } } else { my $stmt = qq{insert into fxprices (currency, date, day_close) values (?, ?, ?);}; print "DatabaseFXDailyData():\n\$stmt = $stmt\n" if $Config{debug}; print "DatabaseFXDailyData(): 1/$ubcfx->{$fx}, $fx, $data{date} \n" if $Config{debug}; if ($Config{commit}) { my $sth = $dbh->prepare($stmt); $sth->execute($fx, $data{date}, 1/$ubcfx->{$fx}) or warn "Failed for $fx at $data{date}\n"; $sth->finish(); } } if ($Config{commit}) { $dbh->commit(); } } } sub UpdateInfoData { # update a row in the info table my ($dbh, %hash) = @_; foreach my $key (keys %hash) { # now split these into reference to the arrays my $cmd = "update stockinfo " . "set capitalisation = $hash{$key}{market_capitalisation}, " . "low_52weeks = $hash{$key}{'52_week_low'}, " . "high_52weeks = $hash{$key}{'52_week_high'}, " . "earnings = $hash{$key}{earnings_per_share}, " . "dividend = $hash{$key}{dividend_per_share}, " . "p_e_ratio = $hash{$key}{price_earnings_ratio}, " . "avg_volume = $hash{$key}{average_volume} " . "where symbol = '$hash{$key}{symbol}';"; $cmd =~ s|'?N/A'?|null|g; # convert (textual) "N/A" into (database) null print "$cmd\n" if $Config{debug}; print "$hash{$key}{symbol} " if $Config{verbose}; if ($Config{commit}) { $dbh->do($cmd) or warn "Failed for $hash{$key}{symbol} with $cmd\n"; } } } sub UpdateTimestamp { my $dbh = shift; my $cmd = q{update beancounter set data_last_updated='now'}; print "$cmd\n" if $Config{debug}; if ($Config{commit}) { $dbh->do($cmd) or warn "UpdateTimestamp failed\n"; $dbh->commit(); } } 1; # required for a package file __END__ =head1 NAME Finance::BeanCounter - Module for stock portfolio performance functions. =head1 SYNOPSIS use Finance::BeanCounter; =head1 DESCRIPTION B provides functions to I, I and I stock market data. I are available of current (or rather: 15 or 20 minute-delayed) price and company data as well as of historical price data. Both forms can be stored in an SQL database (for which we currently default to B though B is supported as well; furthermore any database reachable by means of an B connection should work). I currently consists of performance and risk analysis. Performance reports comprise a profit-and-loss (or 'p/l' in the lingo) report which can be run over arbitrary time intervals such as C<--prevdate 'friday six months ago' --date 'yesterday'> -- in essence, whatever the wonderful B module understands -- as well as dayendreport which defaults to changes in the last trading day. A risk report show parametric and non-parametric value-at-risk (VaR) estimates. Most available functionality is also provided in the reference implementation B, a convenient command-line script. The API might change and evolve over time. The low version number really means to say that the code is not in its final form yet, but it has been in use for well over four years. More documentation is in the Perl source code. =head1 DATABASE LAYOUT The easiest way to see the table design is to look at the content of the B script. It creates the five tables I, I, I, I and I. Note also that is supports the creation of database for both B and B. =head2 THE STOCKINFO TABLE The I table contains general (non-price) information and is index by I: symbol varchar(12) not null, name varchar(64) not null, exchange varchar(16) not null, capitalisation float4, low_52weeks float4, high_52weeks float4, earnings float4, dividend float4, p_e_ratio float4, avg_volume int4 This table is updated by overwriting the previous content. =head2 THE STOCKPRICES TABLE The I table contains (daily) price and volume information. It is indexed by both I and I: symbol varchar(12) not null, date date, previous_close float4, day_open float4, day_low float4, day_high float4, day_close float4, day_change float4, bid float4, ask float4, volume int4 During updates, information is appended to this table. =head2 THE FXPRICES TABLE The I table contains (daily) foreign exchange rates. It can be used to calculate home market values of foreign stocks: currency varchar(12) not null, date date, previous_close float4, day_open float4, day_low float4, day_high float4, day_close float4, day_change float4 Similar to the I table, it is index on I and I. =head2 THE STOCKPORTFOLIO TABLE The I table contains contains the holdings information: symbol varchar(16) not null, shares float4, currency varchar(12), type varchar(16), owner varchar(16), cost float(4), date date It is indexed on I. =head2 THE INDICES TABLE The I table links a stock I with one or several market indices: symbol varchar(12) not null, stockindex varchar(12) not null =head1 BUGS B and B are so fresh that there are only missing features :) On a more serious note, this code (or its earlier predecessors) have been in use since the fall of 1998. Known bugs or limitations are documented in TODO file in the source package. =head1 SEE ALSO F, F, F, F, F =head1 COPYRIGHT Finance::BeanCounter.pm (c) 2000 -- 2006 by Dirk Eddelbuettel Updates to this program might appear at F. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. There is NO warranty whatsoever. The information that you obtain with this program may be copyrighted by Yahoo! Inc., and is governed by their usage license. See F for more information. =head1 ACKNOWLEDGEMENTS The Finance::YahooQuote module by Dj Padzensky (on the web at F) served as the backbone for data retrieval, and a guideline for the extension to the non-North American quotes which was already very useful for the real-time ticker F. =cut beancounter-0.8.10/README.Debian0000644000000000000000000000652310227355363013050 0ustar Debian Installation Notes for beancounter On a Debian system, the following steps are required for beancounter to run: 0. In case you are upgrading from an older installation of beancounter, run the script update_beancounter to add some columns to the tables. 1. Install PostgreSQL [or MySQL or SQLite] root# apt-get install postgresql which will also install postgresql-client and libpgsql2, Configure postgresql with the defaults (ie SQL_ASCII) encoding and ISO datestyle is my preference, others datestyles might or might not work. Postgresql is what I use on my development system. If you have experience with MySQL, use that backend. SQLite may be the very easiest to setup as it is file-based. The preferred SQLite version is now SQLite3 but the previous version is still supported in both setup_beancounter and beancounter itself. 2. Install all the other Perl packages required by beancounter: root# apt-get install libdbd-pg-perl libdbi-perl and the other packages. But 'apt-get install beancounter' should have taken care of that anyway. 3. Configure postgres to let the user as which you are running beancounter create database etc: root# su - postgres postgres:~> createuser -d edd Enter user's postgres ID or RETURN to use unix user ID: 1000 -> Is user "edd" a superuser? (y/n) y WARNING: Any user who can add users can also modify the system catalog createuser: edd was successfully added Note that the superuser rights might not be required, however it is *important* to use the -d option to createuser. 4. Configure postgres to allow TCP/IP connections by uncommenting the following line in /etc/postgresql/postgresql.conf # TCP/IP access is allowed by default, but the default access given in # pg_hba.conf will permit it only from localhost, not other machines. tcpip_socket = true [ Historically, this option was in the file postmaster.init: # PGALLOWTCPIP=no PGALLOWTCPIP=yes but this seems to have changed with the 7.* releases of Postgresql. ] Also ensure that network access is granted via /etc/posgresql/pg_hba_conf: # TYPE DATABASE USER IP-ADDRESS IP-MASK METHOD local all all trust host all all 127.0.0.1 255.0.0.0 trust host all all 192.168.1.0 255.255.255.0 ident sameuser which enables me to have access from the same machine (127.0.0.1) for whoever is on it, as well as from users owning databases from whereever they are on my local LAN. Lastly, restarting the postgresql daemon: root# /etc/init.d/postgresql restart 5. Install beancounter root# dpkg -i beancounter_0.1.0_all.deb Selecting previously deselected package beancounter. (Reading database ... 54112 files and directories currently installed.) Unpacking beancounter (from beancounter_0.1.0_all.deb) ... Setting up beancounter (0.1.0) ... 6. As the user running beancounter, run 'setup_beancounter' to initialise the tables and run an example report. setup_beancounter can create and populate databases for PostgreSQL, MySQL SQLite (the current version 3.*) and SQLite2 (the previous version, with an incompatible format). 7. That's it! beancounter-0.8.10/beancounter_schema_mysql.txt0000644000000000000000000000505210003626251016603 0ustar -- MySQL dump 9.09 -- -- Host: localhost Database: beancounter -- ------------------------------------------------------ -- Server version 4.0.16-log -- -- Table structure for table `beancounter` -- CREATE TABLE beancounter ( version varchar(12) NOT NULL default '', data_last_updated datetime default NULL ) TYPE=MyISAM; -- -- Table structure for table `cash` -- CREATE TABLE cash ( name varchar(16) NOT NULL default '', value float default NULL, currency varchar(12) default NULL, type varchar(12) default NULL, owner varchar(16) default NULL, cost float default NULL, date date default NULL, UNIQUE KEY cash_pkey (name,type,owner,date) ) TYPE=MyISAM; -- -- Table structure for table `fxprices` -- CREATE TABLE fxprices ( currency varchar(12) NOT NULL default '', date date default NULL, previous_close float default NULL, day_open float default NULL, day_low float default NULL, day_high float default NULL, day_close float default NULL, day_change float default NULL, UNIQUE KEY fxprices_pkey (currency,date) ) TYPE=MyISAM; -- -- Table structure for table `indices` -- CREATE TABLE indices ( symbol varchar(12) NOT NULL default '', stockindex varchar(12) NOT NULL default '' ) TYPE=MyISAM; -- -- Table structure for table `portfolio` -- CREATE TABLE portfolio ( symbol varchar(16) NOT NULL default '', shares float default NULL, currency varchar(12) default NULL, type varchar(16) default NULL, owner varchar(16) default NULL, cost float default NULL, date date default NULL, UNIQUE KEY portfolio_pkey (symbol,owner,date) ) TYPE=MyISAM; -- -- Table structure for table `stockinfo` -- CREATE TABLE stockinfo ( symbol varchar(12) NOT NULL default '', name varchar(64) NOT NULL default '', exchange varchar(16) NOT NULL default '', capitalisation float default NULL, low_52weeks float default NULL, high_52weeks float default NULL, earnings float default NULL, dividend float default NULL, p_e_ratio float default NULL, avg_volume int(11) default NULL, active tinyint(1) default '1', PRIMARY KEY (symbol) ) TYPE=MyISAM; -- -- Table structure for table `stockprices` -- CREATE TABLE stockprices ( symbol varchar(12) NOT NULL default '', date date default NULL, previous_close float default NULL, day_open float default NULL, day_low float default NULL, day_high float default NULL, day_close float default NULL, day_change float default NULL, bid float default NULL, ask float default NULL, volume int(11) default NULL, UNIQUE KEY stockprices_pkey (symbol,date) ) TYPE=MyISAM; beancounter-0.8.10/README.non-gnu0000644000000000000000000001271710013034565013240 0ustar Non-GNU Installation Notes for beancounter (especially applicable for Solaris 2.x systems) Systems that don't have bash at /bin/sh and other non GNU utils may find the following guidance helpful. Such systems include SUN Solaris 2.x, and others that derive from the AT&T SVR4 UNIX baseline and many non-Linux systems. The general installation sequence remains unchanged. 00. Read all the information files: README, beancounter.html and BeanCounter.html 0. Install PostgreSQL 1. Configure PostgreSQL (create user(s) and allow TCP/IP db connections) 2. Start postgres database server daemon 3. Build, test and install all beancounter dependent perl modules ( Date::Manip(5.35), DBI(1.16), DBD::Pg(0.93), HTML::Parser(2.20), HTTP::Request(1.23), LWP::UserAgent(1.62), Statistics::Descriptive(2.4), Text::ParseWords(3.1), Finance::YahooQuote, Data::Dumper, Getopt::Long ) 4. perl -w Makefile.PL 5. setup_beancounter # performs the initial db creation and config 6. update_beancounter # performs required updates for new db schema 7. beancounter ready for your use, re-read usage instructions in beancounter.html and BeanCounter.html NB: step 6. is REQUIRED even on an initial beancounter installation (yes, it should be called automatically by setup_beancounter at step 5). If, at step 5, you get message like: setup_beancounter: syntax error at line 37: `progname=$' unexpected You aren't running a sh that understands bash syntax. You will need to get and install bash (on your own) or use the korn shell (ksh) that ships with SUN Solaris 2.x. To use ksh, the first line of each file setup_beancounter and update_beancounter should be changed to: #!/bin/ksh -e If, at step 5, you get the database error message like: *** Error: No postgresql user 'ras' We were unable to start psql as the user 'ras' does not exist You need to create a Postgresql user 'ras' first: Change to user postgres: $ su - postgres Create the user: # createuser ras -d -A Exit from user postgres: # exit and then run this script again. You need to use the following postgresql create user command: /path-to-pgsql-bin/createuser ras -d -A If you get an error from createuser like: createuser: invalid option: -d You are encountering the command line parameter 'out of order' problem. Please, send an email to Dirk about this if only to note the fact that this condition does occur in environments other than mine ;-). Please indicate your OS and shell in the message. Re-ordering the arguments as follows will solve it: /path-to-pgsql-bin/createuser -d -A ras NB: this situation will likely occur in the setup_beancounter and update_beancounter scripts as well, you can either make the changes listed below now, or fix each error one step at a time ... Then repeat step 5. If you continue to get the database error message above and are sure the user id is valid and the database daemon is running or if, at step 5, you get an error from psql like: psql: FATAL: user "-l" does not exist or psql: FATAL: user "-c" does not exist or psql: FATAL: user "-q" does not exist You are encountering the parameter 'out of order' problem within the script. Both scripts will need changes as follows: For script setup_beancounter (3 places): change all occurrences of sequences like '$DBCOMMAND dbname -f' to sequences like '$DBCOMMAND -f dbname' where -f is the (all if multiple) flag option (-l, -q, ...) dbname are parameters without a leading '-' (template1, beancounter, ...) There are 3 occurrences including the help message described above. For script update_beancounter, the correction is a bit different: Change the line: query="select distinct symbol from stockinfo where active" to: query="\t select distinct symbol from stockinfo where active" also change the line: echo $query | $DBCOMMAND -t | grep -q "\b[[:digit:]]" to: echo $query | $DBCOMMAND | grep -q "\b[[:digit:]]" This change is using the psql \t meta-command instead of the psql -t command line argument. Consider sending an email to Dirk about your need to do this if only to note the fact that this condition does occur in environments other than mine ;-). Please indicate your OS and shell in the message. If, when running update_beancounter at step 6, you get a message like: grep: illegal option -- q Usage: grep -hblcnsviw pattern file . . . You are using a non POSIX compliant grep. A GNU grep will work, as will the SUN Solaris grep at /usr/xpg4/bin. The SUN Solaris grep at /usr/5bin will not work. If, after giving it some personal sweat equity, you run into problem(s) that you are unable to resolve by yourself contact Dirk or myself, we will try to help you resolve them. Dirk is the key for anything that relates to beancounter, including the database. I, on the other hand can assist you with script portability issues on non-GNU systems such as Solaris and other ATT based UNIX systems. The assistance pleading message should include: the command line and all resulting output plus as much information about your environment, operating system and as much other info you deem appropriate for remote problem analysis and diagnosis. aloha, ras ps -- constructive comments regarding this text, including variations due to other operating systems and the like are welcome. We prefer diffs created via 'diff -ub orig change' (but will accept diff -cb for those Solaris users that don't have the GNU diff) -- ras beancounter-0.8.10/debian/0000755000000000000000000000000011405255236012220 5ustar beancounter-0.8.10/debian/compat0000644000000000000000000000000110701057405013411 0ustar 5beancounter-0.8.10/debian/changelog0000644000000000000000000007200011405255236014071 0ustar beancounter (0.8.10) unstable; urgency=low * Minor bugfix releases: o manual page confused --dbname and --dbsystem (Closes: #573235) o applied patch by Warren Thompson to use the registered default currency rather than a hard-code 'USD' * debian/control: Standards-Version: increased to 3.8.4 * debian/source/format: Added with "3.0 (native)" * debian/rules: Updated Perl invocation -- Dirk Eddelbuettel Sun, 13 Jun 2010 17:24:30 -0500 beancounter (0.8.9) unstable; urgency=low * Minor bugfix releases: o DateCalc() now requires an error code variable, so supply one o finally release the OandA fx code (Closes: #532743) * debian/control: Standards-Version: increased to 3.8.3 * debian/copyright: Updated to newer format -- Dirk Eddelbuettel Tue, 22 Dec 2009 20:13:59 -0600 beancounter (0.8.8) unstable; urgency=low * Minor bugfix releases: o [beancounter, BeanCounter.pm]: Add "approx" as third argument to the call of Delta_Format() which seems to be needed now * debian/control: Standards-Version: increased to 3.7.2 -- Dirk Eddelbuettel Wed, 03 Oct 2007 22:24:56 -0500 beancounter (0.8.7) unstable; urgency=low * Minor bugfix releases: o [BeanCounter.pm]: Also adjust close for splits in backpopulation o [contrib/beancounter.spec]: New version contributed by Doug Laidlaw * debian/control: Standards-Version: increased to 3.7.0, no changes needed -- Dirk Eddelbuettel Tue, 2 May 2006 22:04:13 -0500 beancounter (0.8.6) unstable; urgency=low * Minor bugfix releases: o [BeanCounter.pm]: Tolerate undef values in daily price updates o [BeanCounter.pm]: Allow host argument to be set from ~/.beancounterrc o [setup_beanconter,update_beancounter]: Make these bash scripts -- Dirk Eddelbuettel Thu, 23 Mar 2006 21:29:56 -0600 beancounter (0.8.5) unstable; urgency=low * Minor update and fixes thanks to a set of excellent patches contributed by Pieter du Preez: o [BeanCounter.pm, beancounter]: Call finish() on all DBI statement handles, and added some other Perl/SQL code improvement o [BeanCounter.pm, beancounter]: More undefined variable tests o [BeanCounter.pm]: Simplify and generalize DBI connection code o [BeanCounter.pm]: Improve representation of currency code mapping o [beancounter]: Correct second date in backpopulation example o [BeanCounter.pm]: Added checks for defined DBI statement handles -- Dirk Eddelbuettel Wed, 15 Mar 2006 20:54:34 -0600 beancounter (0.8.4) unstable; urgency=low * Minor updates and fixes: o [BeanCounter.pm] Additional check against empty currency name. o [beancounter] New command 'host' to define hostname for database server on command-line (as well as via ~/.beancounterrc). o [BeanCounter.pm] If hostname is equal to the default 'localhost', do not connect via tcp/ip to PostgreSQL or MySQL but just use sockets. This may help new users who are unsure how to make their database engines network-aware. o [beancounter] Advances/retracement display now in 79 columns. -- Dirk Eddelbuettel Sun, 19 Feb 2006 21:41:27 -0600 beancounter (0.8.3) unstable; urgency=low * Minor updates and fixes: o [beancounter] New command 'lspositions' for simple display of all positions, not aggregated. o [setup_beancounter] Volume column in stockprices table is now of type numeric; conversion has to be manual (i.e. dump data; recreate schema, reload data) as this is so tedious to code for all three backend. Also add SP500 index via symbol ^GSPC in example setup to use the volume column with data that exceeds the storage of a four-byte int (the previosu choice). o [BeanCounter.pm] Allow market cap in trillions as sometimes seen (in error) for British stocks quotes in pence. Thanks to Robert A. Schmied for the patch. o [BeanCounter.pm] New function DatabaseHistoricalUCBFX for historical FX backpopulation via the service at the Sauder School of UBC in Vancouver. New function GetFXDatum for a single FX data item. o [BeanCounter.pm] Renamed UpdateFXviaPACIFIC to UpdateFXviaUBC o [beancounter] Renamed switch --pacificfx to --ubcfx to select UBC for FX backpopulation and daily updates o [BeanCounter.pm] Update currency codes for Yahoo! service (e.g, switching CAD's symbol from ^XAD to ^XAY), but still no backpopulation capabilities at Yahoo!, and not entirely satisfied with UBC's service either. o [BeanCounter.pm, beancounter] Minor variable renaming and cleanups. o [debian/control] Standard-Version: now 3.6.2 -- Dirk Eddelbuettel Wed, 18 Jan 2006 14:44:50 -0600 beancounter (0.8.2) unstable; urgency=low * Minor update containing a few extensions and fixes o [beancounter] New command-line option --splitby to split-adjust historical data o [BeanCounter.pm] Additional test adjusted-close series to ensure adjusted close and close series are non-zero. o [BeanCounter.pm] Correction to market-cap display for smaller stocks o [setup_beancounter] Example portfolio simplified to US-only to avoid continued unavailability of FX quotes at Yahoo! o [debian/copyright] Updated source URL to dirk.eddelbuettel.com o [MANIFEST] Include debian/copyright -- Dirk Eddelbuettel Sat, 24 Sep 2005 21:16:40 -0500 beancounter (0.8.1) unstable; urgency=low * Minor update release which adds support for SQLite3 o [setup_beancounter] Updated to use sqlite3 as the new default, but added new option '-o $file' to create databases for the previous release of SQLite (which uses an incompatible format relative to SQLite v3.*). o [beancounter, BeanCounter.pm] Accordingly, updated such that SQLite is still the default (but expects a version 3.* database) yet allows for version 2.* databases with the newly added option --dbsystem=SQLite2 o On the Perl side of things, we now require DBD::SQLite releases 1.0.* or later (which match SQLite v3.*) as the default. Access to older SQLite databases is possible via DBD::SQLite2, the compatibility package. o This setup allows users to continue to use existing (v2) SQLite databases, as well as to continue to create them under this older version should they so desire --- but offers the newer and more featureful version 3.* of SQLite as the new default. o Beancounter users employing either PostgreSQL or MySQL are not affected by this in any form. o [debian/control] Depends updated accordingly -- Dirk Eddelbuettel Thu, 14 Apr 2005 22:37:41 -0500 beancounter (0.8.0) unstable; urgency=low * New release with the following new features: o new command-line options for beancounter: --commit whether data update is written to DB, --equityupdate whether equity data should be updated --pacificfx whether redundant FX source should be used with defaults of 'yes', 'yes' and 'no', respetively o new beancounter command 'lsportfolio' to list current portfolio o new beancounter command 'deactivate' to set arguments to inactive o beancounter.spec: Updated with patch provided by R P Herrold o contrib/schnapp: Added contributed script by Mathias Weidner * Detailed changes o Several of the changes below are due to patches and/or suggestions by Robert A. Schmied and Mathias Weidner. o [beancounter] Checks that --date and --prevdate arguments are valid dates (using the Date::Manip parser) o [beancounter] Support new command-line argument and options; added --help info as well as perldoc documentation o [beancounter] New function lsportfolio for eponymous command o [beancounter] New function inactive_portfolio for 'deactivate' o [beancounter] Output tables slightly reformatted and aligned o [BeanCounter.pm] New functions GetUBCFXData, GetUBCFXHash, UpdateFXviaPacific implementing the redundant FX data sourcing o [BeanCounter.pm] Automatically reflect split-adjusted data in backpop o [BeanCounter.pm] Updated GetConfig function for new options o [BeanCounter.pm] Improved SQL logic in GetCashData function o [BeanCounter.pm] Improved GetFXData to find most current data before or equal to request date to deal better with non-biz days o [BeanCounter.pm] Support --commit argument to database updates o [BeanCounter.pm] Added check for data in DatabaseInfoData(); added function ExistsInfoSymbol to implement the lookup o [BeanCounter.pm] Added simple FX retrieval heuristic (employing the ISO symbol) in ScrubDailyData() o [debian/control] Also 'OR'ed Depends on libdbd-sqlite-perl and sqlite -- Dirk Eddelbuettel Mon, 21 Mar 2005 22:24:31 -0600 beancounter (0.7.6) unstable; urgency=low * Bug fix release - [Makefile.PL]: Added dependency on Finance::YahooQuote which should have been added a long time ago -- thanks, CPAN Testers! - [setup_beancounter]: Fixed path to test version of beancounter - [contrib/getDiv]: Added contributed script by Joao Antunes Costa -- Dirk Eddelbuettel Wed, 28 Jul 2004 22:42:48 -0500 beancounter (0.7.5) unstable; urgency=low * Bug fix release - [beancounter]: Correct documentation of 'passwd' option for db connection from 'password' to 'passwd' (Closes: #255640) - [beancounter]: Add simple command 'checkdbconnection' that exits with testable error code if db connection can be opened + closed - [setup_beancounter]: Make use of 'checkdbconnection' test -- Dirk Eddelbuettel Thu, 24 Jun 2004 18:23:45 -0500 beancounter (0.7.4) unstable; urgency=low * Bug fix release with several contributed patches: - [BeanCounter.pm]: Added portfolio export function GetPortfolioData which will eventually be used by smtm and others, provided by Kevin Kim - [BeanCounter.pm]: Ensure proxy information is set, provided by Joao Costa - [BeanCounter.pm]: Ensure data written to stockprices table has no NAs good data should never be overwritten by bad data, also provided by Joao Costa - [beancounter]: Historical data from non-US exchanges needs to be retrieved in patches of 200, implemented in patch provided by Matthew Jurgens -- Dirk Eddelbuettel Wed, 2 Jun 2004 21:19:52 -0500 beancounter (0.7.3) unstable; urgency=low * Bug fix release: - [BeanCounter.pm] For backpopulation of foreign exchange data, divide all data with dates past Dec 30, 2003, by a factor of 100.0 to account for Yahoo's change from a 'dollars' to 'cents' scale. - [BeanCounter.pm] For backpopulation of prices and fx series, skip over the newly added html comment line Yahoo! decided to throw in there. Thanks to Robert A. Schmied for the heads-up on this. - [Makefile.PL] Be less restrictive and require onky DBI; as any one of four DBD modules could be used, it makes no sense to impose on (Pg) on everybody. - [debian/rules] Integrated 'perl Makefile.PL; make dist' needed for proper CPAN uploads into the normal 'update' from the my sources -- Dirk Eddelbuettel Sat, 3 Apr 2004 09:52:11 -0600 beancounter (0.7.2) unstable; urgency=low * Bug fix release: - [BeanCounter.pm:] Scale FX data retrieved after 20040226 by a factor of 100 to correct for Yahoo's move from dollars to cents. Initial patch by Phil Homewood augmented with Date_Cmp() use. - README.non-gnu: small corrections by Robert A. Schmied - t/01base.t: Added simple and relatively meaningless test at the repeated request of the good folks at CPAN. Beancounter really needs and existing database to work with which we can't assume at build time, so the testing potential is somewhat limited. As we load the module, most of the (run-time) Depends now need to be in Build-Depends. - debian/control: Augmented Build-Depends accordingly -- Dirk Eddelbuettel Tue, 2 Mar 2004 21:22:44 -0600 beancounter (0.7.1) unstable; urgency=low * Bug fix release: - [BeanCounter.pm:] Don't return from GetConfig() when no per-user config file is found but simply skip reading the file. -- Dirk Eddelbuettel Wed, 4 Feb 2004 22:59:57 -0600 beancounter (0.7.0) unstable; urgency=low * New release with the following new features: - [beancounter, BeanCounter.pm, setup_beancounter:] Added complete support for the SQLite database backend - [README.non-gnu:] Documentation on beancounter installation and use on non-GNU standard systems (with emphasis on SUN Solaris) kindly contributed by Robert A. Schmied - [beancounter_schema_{postgresql,mysql,sqlite}.txt:] Include text dumps of database schema in case database has to be created manually - [beancounter_example.txt:] Full session log from setup_beancounter covering database creation, example data insertion, backpopulation and initial example reports * Enhancements and minor bug fixes tripped up mostly by the using SQLite - [beancounter:] day_end_report() now uses explicit request for previous day's fx prices as opposed to the previous_close columns - [beancounter:] added support for SQLite in deletedb() and made system() calls trigger a warning if deletion fails - [beancounter:] added advances and retracement reports to allreports - [BeanCounter.pm:] added support for SQLite in ConnectToDb() - [BeanCounter.pm:] corrected a few 'order by' in SQL join statement by explicitly referring to table and symbol - [BeanCounter.pm:] normalized backpopulation and fxbackpopulation to insert data in %Y%m%d format in which we tend to retrieve and compare - [BeanCounter.pm:] use BeanCounter (not Beancounter) in POD docs - [README.Debian:] Updated throughout -- Dirk Eddelbuettel Mon, 26 Jan 2004 21:57:30 -0600 beancounter (0.6.5) unstable; urgency=low * Bug fix releases with patches from Robert A. Schmied: - [beancounter:] Render display_report a little more robust - [beancounter:] typo correction for Advancement - [BeanCounter.pm:] More explicit warning messages in ScrubDailyData - [BeanCounter.pm:] Corrected a test to ensure volume is stored too on backpopulation - [updates_beancounter, setup_beancounter:] Removes superfluous empty parantheses at the end of function defintions for ksh compatibiliy * New documentation files added just in case: - [beancounter_example.txt:] (from running 'setup_beancounter') - [beancounter_schema_postgresql.txt:] (from 'pgdump --schema-only') - [beancounter_schema_mysql.txt:] (output from 'mysqladmin --no-data') -- Dirk Eddelbuettel Thu, 22 Jan 2004 21:59:36 -0600 beancounter (0.6.4) unstable; urgency=low * Bug fix release: - [BeanCounter.pm:] Applied patch by Thomas Walter which corrects the SQL logic in GetPriceData that applies to restrictions covering stocks appearing in multiple portfolios, which can affect the status report. - [BeanCounter.pm:] Applied another patch by Thomas Walter extending restrictions to other core functions, and adding debugging output - [beancounter:] Call GetCashData() later in portfolio_status() - [beancounter:] Corrected some formatting for reports -- Dirk Eddelbuettel Mon, 22 Dec 2003 19:58:53 -0600 beancounter (0.6.3) unstable; urgency=low * Bug fix release: - [BeanCounter.pm:] Allow six-column format for DatabaseHistoricalFXData to reflect a change in the Data returned by Yahoo! (Closes: 222408) - [beancounter:] Small changes to better align reports on 79 columns -- Dirk Eddelbuettel Fri, 28 Nov 2003 21:33:16 -0600 beancounter (0.6.2) unstable; urgency=low * Bug fix release: - [BeanCounter.pm, setup_beancounter, update_beancounter:] Default value of active field changed from 't' to '1' - [BeanCounter.pm:] DB Schema comparison now based on numeric value - [BeanCounter.pm:] If no FX data found, suggest --date/--prevdate option - [beancounter: Add uppercase'd symbol to portfolio - [beancounter, setup_beancounter, update_beancounter:] Use distinct version number for db schema comparison - [setup_beancounter, update_beancounter:] Use 'timestamp with time zone' for beancounter when PostgreSQL is used - [setup_beancounter:] Use $USER, not $user, for error message -- Dirk Eddelbuettel Tue, 6 May 2003 21:16:52 -0500 beancounter (0.6.1) unstable; urgency=low * Bug fix release: - [BeanCounter.pm:] Force uppercase'ing of stock symbols prior to storing either daily or historical records in the database. - [debian/rules:] Include flip_symbol.sh in examples/ dir -- Dirk Eddelbuettel Mon, 30 Dec 2002 21:41:01 -0600 beancounter (0.6.0) unstable; urgency=low * New release with the following new features: - Switched to using Finance::YahooQuote (>= 0.18) for quote gathering after code we had here has been consolidated in the module. This may require adjusting of (numeric) stock symbols for which updated_beancounter attempts to test. - New command 'fxbackpopulate' to grab historical currency data from Yahoo!. With this added capability, portfolios with non-home country stocks can run reports for profit/loss or risk without having to built the database over time. The major bummer is that Yahoo! stores historic currency data in a braindead format of only two decimals so we get suboptimal accuracy. - Changed setup_beancounter to transform example portfolio into one composed of one stock each from US, CA, FR and DE. Backpopulates for prices and currencies and runs p/l and risk reports right out of the box. - Simplified quote gathering code in BeanCounter.pm; requests for all markets can go against the main Yahoo! server. - Applied patch by Ken Neighbors to use a different Yahoo! source URL for historic prices, correct a one-off error in date / month conversion and correct documentation. * Detailed changes - [beancounter:] New command (and function) 'fxbackpopulate' - pbeancounter:] Corrected small mistakes in documentation via Ken's patch, added documentation and examples re 'fxbackpopulate' - [BeanCounter.pm:] Ken's patch re other URL, and one-off month error - [BeanCounter.pm:] New function DatabaseHistoricalFXData - [BeanCounter.pm:] Commented out a lot of code no longer required as all current price data can be had from one main Yahoo! source - [BeanCounter.pm:] Ensure new rows in 'stockinfo' table are set to active - [BeanCounter.pm:] Use Finance::YahooQuote, delete code thus made redundant - [setup_beancounter:] Updated HP to HPQ (from HWP) in Dow example - [setup_beancounter:] Changed example portfolio from two US stocks to one each from US,CA,FR,DE, added fxbackpopulate call for CAD and EUR - [setup_beancounter:] Smartened up the use of 'last business day' for the example portfolio: on weekends or Monday we use 'last friday', on other days we use 'yesterday' and made sure all example reports use this date -- Dirk Eddelbuettel Sun, 29 Dec 2002 20:01:06 -0600 beancounter (0.5.1) unstable; urgency=low * [setup_beancounter:] Fill beancounter table with current version number and current date right after the table is created. * [debian/rules:] Making sure $version is updated in beancounter, update_beancounter and setup_beancounter -- Dirk Eddelbuettel Sun, 10 Mar 2002 19:38:57 -0600 beancounter (0.5.0) unstable; urgency=low * New release with the following new features: - New command 'retracement' to calculate drawdowns for the given data period -- these are defined as price decreases relative to the maximum price in the period. This can be seen as hypothetical unrealized losses relative to the would-coulda-shouda optimal selling price. Shorts are treated the other way relative to their lows. - New command 'advancement' which does the same for gains relative to lows. - Applied patch by Peter Kim so that 'update' now batches queries in round lots of one hundred symbols. - The backpopulate command is now idempotent too: it can now be run even if data already exists and will use either 'insert' or 'update' as required. - The stockinfo table has a new column 'active' allowing to flag inactive stocks (or e.g. expired options) with a 'false' value. - The database has a new table 'beancounter' with fields for the current version (to ensure code and database schema match) and the most recent update. * Detailed changes - [BeanCounter.pm:] Applied Peter's patch, pluse comments / indents - [BeanCounter.pm:] New function GetRetracementData - [BeanCounter.pm:] New function UpdateTimestamp - [beancounter:] New function portfolio_retracement - [BeanCounter.pm:] Rewrote DatabaseHistoricalData for idempotency - [beancounter:] Example now uses full example with purchase date / time - [BeanCounter.pm:] Rewrote SQL restriction for getting price data - Database scheme:] New field 'active' to flag inactive stocks / options - [BeanCounter.pm:] Make use of 'active' field when retrieving data - [BeanCounter.pm:] Simpliefied API for ExistsDailyData() - [setup_beancounter:] Added creation of active field in stockinfo - [update_beancounter:] Idem * debian/control: Spelling correction (Closes: #124442) * debian/control: Upgraded to Standards-Version 3.5.0 -- Dirk Eddelbuettel Tue, 5 Mar 2002 22:06:51 -0600 beancounter (0.4.0) unstable; urgency=low * New release with the following new features - Support for MySQL has been added: beancounter can now work with both PostgreSQL and MySQL, including all steps up from the database creation - The update command's SQL query has been rewritten so that all stocks known to beancounter are updated, yet still allows the use of the --restriction option to specify only subsets of the portfolio - The update command is now idempotent: it can now be run several times during a trading day; the initial data set will be 'insert'ed, following ones 'update'd (this comes at the cost of an additional query) - Better support of old, expired or delisted stocks: unparseable dates, as well as dates that "too old" (where the current default 5 business days) are set to N/A; and dates which test for N/A are neither scrubbed nor databased during the "update" command. - Database backend and database name can now be set as parameters which facilitates multiple databases and backends to be used in parallel - New (boolean) option --fxupdate which defauls to 'true', hence the use of --nofxupdate prevents any updates to the FX database - Database scheme change in stockprices table: column names changed to day_open, day_close, day_change for consistency (and MySQL gripes). Similarly, index becomes stockindex in the indices table. Hence, users /upgrading from an older version/ must run update_beancounter. * Detailed changes: - [BeanCounter.pm] --verbose and --debug now passed to DBI connection for Warn and PrintError, respectively - [BeanCounter.pm] New configuration options dbsystem and dbname, defaulting to PostgreSQL and beancounter - [BeanCounter.pm] Protect name with dbh->quote for DatabaseInfoData - [BeanCounter.pm] DatabaseUpdate has new SQL query with sub-select - [BeanCounter.pm] Several small code cleanups and documentation updates - [beancounter] New command-line options --dbsystem and --dbname - [beancounter] Several updates and extensions to the pod documentation - [setup_beancounter] Now with command-line options -m, -s dbname, -s - [setup_beancounter] Parallel code for MySQL db creations added - [setup_beancounter] Added manual page as POD document within - [update_beancounter] Added code for the database table transition - [update_beancounter] Added manual page as POD document within -- Dirk Eddelbuettel Sun, 14 Oct 2001 20:57:05 -0500 beancounter (0.3.1) unstable; urgency=low * Bug fix release: - rewrote SQL queries using '--restriction arg' to allow for multiple portfolio restrictions to be imposed (BeanCounter.pm) - reflect change at Yahoo! Europe and convert quotes from ';' field seperator and ',' decimal point (BeanCounter.pm: GetQuote) - ensure that day_end_report is relative to previous day (beancounter) - add dbh->commit() after database updates (beancounter: add_index, add_portfolio; BeanCounter.pm: delete_stock) * Some other minor changes: - annualise returns if stock held > 1 year (beancounter: display_status) - make some functions more compact with selectcol_arrayref (BeanCounter.pm) - use dropdb(1) to delete database (requires PostgreSQL 7.*) (beancounter) - add warning if correlation non computable (BeanCounter.pm: GetRiskData) - add warning if date not parseable (BeanCounter: ParseDailyData) - add historica price retrieval for (US) mutual funds (BeanCounter.pm: GetHistoricalData) - if data inconsistent, adjust close to to previous_close plus change (BeanCounter: SrubDailyData) -- Dirk Eddelbuettel Mon, 13 Aug 2001 21:53:18 -0500 beancounter (0.3.0) unstable; urgency=low * New release with the following code changes - new command "risk" for report with value-at-risk at (VaR) percentile estimates, using both the standard (parametric) approach as well as a nonparametric quantile estimate "risk" also computes marginal value-at-risk (slow for large portfolios) "risk" requires the Statistics::Descriptive modules - new option "--forceupdate " to permit overriding of a faulty date supplied by Yahoo! Finance (which is becoming more common) - new command "allreports" for dayend, status and risk reports all in once - added status and risk reports to jobs ran by "dailyjob" command - new ScrubDailyData routine; currently only using the "--forceupdate date" check on the pricing date supplied by Yahoo! - prevdate defaults to "six months ago" (better default for risk report) - "status" also shows cash holding from table `cash' (preliminary feature) - improved internal logic for argument and option checking + db connection - improved report calculations, report display and documentation - documentation corrected (command is 'delete', not 'deletestock') -- Dirk Eddelbuettel Thu, 29 Mar 2001 19:18:54 -0600 beancounter (0.2.1) unstable; urgency=low * Bug fix release: - setup_beancounter: change last fromdate,todate to prevate,date - setup_beancounter: use NT, not NT.TO, to avoid FX in demo - beancounter: Use psql, not destroydb or dropdb, to delete db - debian/control: Also depend on libdbd-pg-perl (Closes: #85363) -- Dirk Eddelbuettel Fri, 9 Feb 2001 22:45:54 -0600 beancounter (0.2.0) unstable; urgency=low * New release with the following code changes - new table columns 'owner' and 'holder' for portfolio allow to differentiate between different accounts and users - SQL restrictions can be imposed for finer-grained analysis - new status command for portfolio return and holding overview - extrafx argument allows to specify additional currencies to be downloaded and stored by the 'update' command - date and prevdate default to today and yesterday - fromdate and todate options replaced by consistent use of prevdate and date - more documentation - data display layout changed -- Dirk Eddelbuettel Sat, 2 Dec 2000 17:27:48 -0600 beancounter (0.1.1) unstable; urgency=low * Bugfix release: - BeanCounter.pm: corrected minor typo in DatabaseInfoData - debian/rules: don't create beancounter.1 -- Makefile.PL does that too -- Dirk Eddelbuettel Tue, 25 Jul 2000 22:47:40 -0400 beancounter (0.1.0) unstable; urgency=low * Initial Debian (and upstream) release. -- Dirk Eddelbuettel Sun, 23 Jul 2000 22:14:17 -0400 beancounter-0.8.10/debian/rules0000755000000000000000000001036311405254734013305 0ustar #! /usr/bin/make -f # -*- makefile -*- # debian/rules file for the Debian/GNU Linux beancounter package # Copyright (C) 2001 - 2010 by Dirk Eddelbuettel package := $(shell grep Package debian/control | sed 's/^Package: //') version := $(shell head -1 debian/changelog | \ perl -nle 'm/\S+\s+\((\S+)\)/ && print $$1') debtmp := $(shell pwd)/debian/$(package) srcdir := ../../../progs/perl/beancounter webdir := /home/edd/www/code/beancounter host := $(shell hostname) # Uncomment this to turn on verbose mode. #export DH_VERBOSE=1 update: ifneq ($(host),max) @echo "This needs to run on max" false endif cp -vaf $(srcdir)/BeanCounter.pm \ $(srcdir)/beancounter \ $(srcdir)/README \ $(srcdir)/README.Debian \ $(srcdir)/README.non-gnu \ $(srcdir)/TODO \ $(srcdir)/THANKS \ $(srcdir)/Makefile.PL \ $(srcdir)/example.beancounterrc \ $(srcdir)/beancounter_example.txt \ $(srcdir)/beancounter_schema_postgresql.txt \ $(srcdir)/beancounter_schema_mysql.txt \ $(srcdir)/beancounter_schema_sqlite.txt \ $(srcdir)/setup_beancounter \ $(srcdir)/update_beancounter \ $(srcdir)/contrib/ \ $(srcdir)/t/ . @echo "********* Now run updatebuild on max" updatebuild: # update the $$version field in the perl code as well as in # the two support scripts perl -p -i -e \ "s/version = \".*\";/version = \""$(version)"\";/" $(package) for i in setup_beancounter update_beancounter; do \ perl -p -i -e "s/VERSION='.*'/VERSION='"$(version)"'/" $$i; \ done perl -p -i -e \ "s/'VERSION' \t=> '\d.\d.\d'/'VERSION'\t\t=> '"$(version)"'/"\ Makefile.PL perl Makefile.PL make dist mv -v Finance-BeanCounter-$(version).tar.gz .. webdir: ifneq ($(host),max) @echo "This needs to run on max" false endif cp -vax /var/cache/pbuilder/result/$(package)_$(version)_all.deb .. (cd ..; cp $(package)_$(version)_all.deb /tmp; \ cd /tmp; \ fakeroot alien -r $(package)_$(version)_all.deb; \ fakeroot mv $(package)-$(version)-2.noarch.rpm \ $(package)-$(version).noarch.rpm; \ cd - ; \ mv /tmp/$(package)-$(version).noarch.rpm . ) cp -af ../$(package)_$(version).tar.gz \ ../$(package)_$(version)_all.deb \ ../$(package)-$(version).noarch.rpm \ TODO \ debian/changelog \ beancounter.html \ BeanCounter.html $(webdir) (cd $(webdir); \ mv changelog ChangeLog; \ ln -sfv $(package)_$(version).tar.gz \ $(package)-current.tar.gz ; \ ln -sfv $(package)_$(version)_all.deb \ $(package)-current.deb ; \ ln -sfv $(package)-$(version).noarch.rpm \ $(package)-current.rpm ; \ ln -sfv ChangeLog ChangeLog.txt ; \ ln -sfv TODO TODO.txt ) build: build-stamp build-stamp: dh_testdir #$(PERL) Makefile.PL $(config) perl Makefile.PL INSTALLDIRS=vendor pod2html --flush $(package) > $(package).html pod2html --flush BeanCounter.pm > BeanCounter.html #pod2man $(package) > $(package).1 $(MAKE) $(MAKE) test touch build-stamp clean: dh_testdir dh_testroot rm -f build-stamp .html -test -f Makefile && $(MAKE) realclean -rm -f pod2html-itemcache pod2html-dircache pod2htmi.tmp pod2htmd.tmp dh_clean binary-indep: build dh_testdir dh_testroot dh_clean -k #dh_installdirs usr/bin usr/share/man/man1 #dh_installdirs usr/bin #$(MAKE) prefix=$(debtmp)/usr pure_install $(MAKE) install DESTDIR=$(debtmp) dh_perl dh_installdocs THANKS TODO README README.Debian \ README.non-gnu dh_installexamples example.beancounterrc flip_symbol.sh \ beancounter_example.txt \ beancounter_schema_postgresql.txt \ beancounter_schema_sqlite.txt \ beancounter_schema_mysql.txt \ contrib/ #dh_installmenu #dh_installinit #dh_installcron dh_installman #dh_undocumented setup_beancounter.1 update_beancounter.1 dh_installchangelogs dh_compress dh_fixperms #dh_suidregister dh_installdeb dh_gencontrol dh_md5sums dh_builddeb binary-arch: build source diff: @echo >&2 'source and diff are obsolete - use dpkg-source -b'; false binary: binary-indep binary-arch .PHONY: build clean binary-indep binary-arch binary beancounter-0.8.10/debian/source/0000755000000000000000000000000011405255016013514 5ustar beancounter-0.8.10/debian/source/format0000644000000000000000000000001511405255016014723 0ustar 3.0 (native) beancounter-0.8.10/debian/copyright0000644000000000000000000000072111314277071014153 0ustar This is the Debian GNU/Linux version of the beancounter stock portfolio performance monitoring program. This package was both written and put together for Debian by Dirk Eddelbuettel . The sources can also be obtained from http://dirk.eddelbuettel.com/code/beancounter.html Copyright (C) 1998 - 2009 Dirk Eddelbuettel License: GPL-2 On a Debian GNU/Linux system, the GPL-2 license is included in the file /usr/share/common-licenses/GPL-2. beancounter-0.8.10/debian/control0000644000000000000000000000205711405254622013625 0ustar Source: beancounter Section: misc Priority: optional Maintainer: Dirk Eddelbuettel Standards-Version: 3.8.4 Build-Depends-Indep: libdate-manip-perl, libstatistics-descriptive-perl, libfinance-yahooquote-perl, libdbi-perl Build-Depends: debhelper (>= 5.0.0) Package: beancounter Architecture: all Depends: ${misc:Depends}, ${perl:Depends}, libfinance-yahooquote-perl (>= 0.18), libdate-manip-perl, libdbi-perl, libdbd-pg-perl | libdbd-mysql-perl | libdbd-odbc-perl | libdbd-sqlite3-perl | libdbd-sqlite2-perl, libstatistics-descriptive-perl, postgresql-client | mysql-client | sqlite3 | sqlite Description: A stock portfolio performance monitoring tool This package provides beancounter, a tool to quantify gains and losses in stock portfolios, as well as the BeanCounter Perl module that underlies it. Beancounter queries stock prices from Yahoo! Finance server(s) around the globe and stores them in a relational database (using PostgreSQL) so that the data can be used for further analysis. Canned performance and risk reports are available. beancounter-0.8.10/beancounter_example.txt0000644000000000000000000001022110003627551015547 0ustar Script started on Wed 21 Jan 2004 08:00:37 PM CST edd@homebud:~> setup_beancounter Creating beancounter database ** Running: createdb beancounter CREATE DATABASE Creating beancounter database tables Filling beancounter database tables with DJIA stocks ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter addindex DJIA AA AXP T BA CAT C KO DIS DD EK XOM GE GM HPQ HD HON INTC IBM IP JNJ MCD MRK MSFT MMM JPM MO PG SBC UTX WMT Filling beancounter (sample) portfolio ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter addportfolio CSCO:50:USD NT.TO:100:CAD SIEGn.DE:10:EUR CGEP.PA:50:EUR Filling beancounter with stock info and most recent prices for DJIA stocks ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter addstock AA AXP T BA CAT C KO DIS DD EK XOM GE GM HPQ HD HON INTC IBM IP JNJ MCD MRK MSFT MMM JPM MO PG SBC UTX WMT Filling beancounter with historical prices for example portfolio stocks ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter backpopulate --prevdate '1 year ago' --date 'yesterday' CSCO NT.TO SIEGn.DE CGEP.PA adding CSCO from 20030121 to 20040120 adding NT.TO from 20030121 to 20040120 adding SIEGN.DE from 20030121 to 20040120 adding CGEP.PA from 20030121 to 20040120 Filling beancounter with historical fx prices for EUR and CAD ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter fxbackpopulate --prevdate '1 year ago' --date 'yesterday' EUR CAD backpopulating EUR (using ^XEU) from 20030121 to 20040120 backpopulating CAD (using ^XCD) from 20030121 to 20040120 Running portfolio pl report on (sample) portfolio ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter plreport --date 'yesterday' =============================================================================== Profit / loss from 21 Jul 2003 to 20 Jan 2004 abs, rel change ------------------------------------------------------------------------------- ALCATEL USD 437.31 7.74 852.39 13.53 415.08 94.92% CISCO SYSTEMS USD 896.50 17.93 1443.00 28.86 546.50 60.96% NORTEL NETWORKS USD 301.75 4.25 702.00 9.00 400.25 132.64% SIEMENS N USD 524.55 46.42 851.26 67.56 326.71 62.28% ------------------------------------------------------------------------------- Grand Total USD 2160.11 3848.65 1688.54 78.17% =============================================================================== Running portfolio riskreport on (sample) portfolio ** Running: beancounter --dbsystem PostgreSQL --dbname beancounter risk --date 'yesterday' =============================================================================== Portfolio Risk on 20 Jan 2004 going back to 21 Jul 2003 ------------------------------------------------------------------------------- Name Position 1% Profit/Loss Volatility VaR margVaR =============================================================================== ALCATEL USD 852.39 -5.1% -43 40.2% -50 -33 CISCO SYSTEMS USD 1443.00 -4.4% -62 30.8% -65 -43 NORTEL NETWORKS USD 702.00 -6.8% -47 58.1% -59 -37 SIEMENS N USD 851.26 -4.4% -37 26.6% -33 -20 ------------------------------------------------------------------------------- Portfolio level USD 3848.65 27.6% -155 ------------------------------------------------------------------------------- Portfolio VaR is 4.0% of assets, or 74.8% of VaR sum of -208 =============================================================================== VaR calculations use a 99% confidence level and 1-day horizon. Marginal VaR is the change to the portfolio VaR attributable to adding this position. Computing the 1% quintile of the return distribution, which can be viewed as a non-para- metric VaR estimate, requires at least 100 observations. =============================================================================== Done. edd@homebud:~> exit Script done on Wed 21 Jan 2004 08:01:18 PM CST beancounter-0.8.10/TODO0000644000000000000000000001020707557035273011500 0ustar * stock splits [ idempotent backpopulation, which is done in 0.5.0, eases this a little ] * annualised returns [was done in status command, currently turned off; probably needs another switch to allow a date after which we annualise, say a year] [turned back on again] * clean up code * more error checking on calls to DBI et al * read more info from the rc file ~/.beancounterrc such as host, port, user, ... [ mostly done; this grows organically on an as-needed basis ] * allow "restrictions" on beancounter operations, eg do a portfolio update at 13:00 EST with the restriction of doing it only for shares with currencies that are GBP or EUR (or SEK or DKK ...) to get the European market [ done in 0.2.0, also added 'extrafx' argument ] * there is buglet that rears its head when a stock is added to portfolio during business hours as its current price is also added for that day ... which creates a conflict when that stock is later updated during the normal update. This is difficult to circumvent as we don't want to reinvent cron within beancounter. Either we do update for the current day (as we do now, which has the problem) or don't (in which case we might never get that datum if we updated after the daily cron run) [ cf below on idempotent operations, should no longer be an issue ] * fx quotes are currently retrieved via Yahoo from the Philadelphia exchange -- but that means no quotes are available on US holidays. * We also have no access to historical FX data for backpopulation [ as of 0.6.0 we do, albeit at a grizzly two-decimals precision ] * there is also a plain error somewhere as eg a DEM based portfolio sees a return even though a EUR traded stock didn't move -- can't really happen as DEM/EUR rates are fixed. Maybe add an crosscurrency table/hash and do no ask Yahoo! if cross-rate known [ not an issue if portfolio is reported in EUR ] * Yahoo seems to return bad date information every now and then. A "scrubbing" between retrieval and databasing might help: we can check if other data points for the same exchange have a different date and then use this as an override. Might be overkill. [ partly done; now have --forcedate option ] * Allow for cash holdings [ partly done; needs support for adding/changing/deleting positions ] * Options (.X) volume adjustment (ie an option contract is over 100 shares) * MySQL support [ added in 0.4.0 following Mathias' patch with the initial MySQL schema ] * Beancounterrc manual page [ there is an updated section in the beancounter(1) manual page ] * Idempotent operation, ie call update several times a day [ added in 0.4.0 based on Mathias' patch ] * Versioning of db schema, ie add new table "version" with one column and maybe a date * update_beancounter needs to reflect the post-0.3.1 table changes [ done, tested against 0.1.1, 0.3.0 and 0.3.1 (with PostgreSQL) ] * beancounter regression test data set, data "feeding" program and test routines [ Mathias is looking into this ] * updates for stocks in stockinfo but not in portfolio; maybe via an option to update (eg --fullupdate) [ done in 0.4.0 via rewriting of the symbol selection SQL query ] * new command for partial deletes (ie from portfolio, but leave stockinfo and stockprices). Or maybe make partial delete the default, and only nuke from stockinfo/stockprices if --purge option given * GUI: -- one choice would be via perl/Tk [ but IMHO Tk's canvas widget is too primitive ] -- another via R's (www.r-project.org) tcltk package; R's advanced graphics and analysis functions could shine there [ have prototypes ] -- web-based via mod_perl [ Mathias' suggestion ] * (really distant wishlist feature) build portfolio up from stock trades, have analysis driven from transactions * new columns "soldprice","solddate" to compute returns on completed transactions * suppress noisy error messages when we have not enough data for the risk report * added 'active' boolean field to stockinfo [ done in 0.5.0 ] * restrictions don't aggregate right if they only "partly" hit a holding (eg holding a stock in two accounts, blocking one --> still get total) beancounter-0.8.10/beancounter0000755000000000000000000020431411405254334013233 0ustar #!/usr/bin/perl -w # # beancounter --- A stock portfolio performance monitoring tool # # Copyright (C) 1998 - 2010 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # $Id: beancounter,v 1.92 2010/06/13 21:55:54 edd Exp $ # adjust @INC to let the current development version be found first BEGIN { @INC = ( ".", @INC ) } use strict; # be careful out there, son use Date::Manip; # general date parsing / calcs use English; # explicit variable names use Getopt::Long; # long options use vars qw($help $debug $verbose $fxarg $datearg $prevdatearg $rcfile $sqlrestriction $extrafx $updatedate $dbsystem $dbname $fxupdate $commit $equityupdate $hostarg $ubcfx $splitby); use Finance::BeanCounter; # beancounter functions my $rcsversion = sprintf("%d.%d", q$Revision: 1.92 $ =~ /(\d+)\.(\d+)/); my $version = "0.8.10"; # updated from the debian/rules Makefile my $db_min_schema = "0.6.0"; # minimum version of the database that we need my $date = # inner expression below is updated by RCS sprintf("%s", q$Date: 2010/06/13 21:55:54 $ =~ /\w*: (\d*\/\d*\/\d*)/); my $rcfile = $ENV{HOME} . "/.beancounterrc"; ($prevdatearg, $datearg, $equityupdate, $fxupdate, $commit, $ubcfx, $splitby, $hostarg) = ("6 month ago", "today", 1, 1, 1, 0, 2, "localhost"); my %options = ("help" => \$help, "debug" => \$debug, "currency=s" => \$fxarg, "date=s" => \$datearg, "prevdate=s" => \$prevdatearg, "restriction=s" => \$sqlrestriction, "dbsystem=s" => \$dbsystem, "dbname=s" => \$dbname, "extrafx=s" => \$extrafx, "rcfile=s" => \$rcfile, "commit!" => \$commit, "forceupdate=s" => \$updatedate, "fxupdate!" => \$fxupdate, "equityupdate!" => \$equityupdate, "splitby=s" => \$splitby, "host=s" => \$hostarg, "ubcfx!" => \$ubcfx, "verbose" => \$verbose); help_and_exit() if (!GetOptions(%options) or $help or $#ARGV < 0); $OUTPUT_AUTOFLUSH = 1; my $command = shift @ARGV; my %Config = GetConfig($rcfile, $debug, $verbose, $fxarg, $extrafx, $updatedate, $dbsystem, $dbname, $fxupdate, $commit, $equityupdate, $ubcfx, $hostarg, $command); my $dbh = ConnectToDb(); if (TestInsufficientDatabaseSchema($dbh, $db_min_schema)) { warn "Database schema is not current. Please run 'update_beancounter'\n"; } elsif (!ParseDate($datearg)) { warn "Please correct the invalid --date argument $datearg.\n"; } elsif (!ParseDate($prevdatearg)) { warn "Please correct the invalid --prevdate argument $prevdatearg.\n"; } else { if ($command =~ /^plreport$/) { portfolio_report($sqlrestriction); } elsif ($command =~ /^backpopulate$/) { backpopulate(@ARGV); } elsif ($command =~ /^fxbackpopulate$/) { fxbackpopulate(@ARGV); } elsif ($command =~ /^addindex$/) { add_index(@ARGV); } elsif ($command =~ /^addstock$/) { add_stock(@ARGV); } elsif ($command =~ /^addportfolio$/) { add_portfolio(@ARGV); } elsif ($command =~ /^advancement$/) { portfolio_retracement($sqlrestriction, "advance"); } elsif ($command =~ /^allreports$/) { day_end_report($sqlrestriction); portfolio_status($sqlrestriction); portfolio_risk($sqlrestriction); portfolio_retracement($sqlrestriction, "advance"); portfolio_retracement($sqlrestriction, "retrace"); } elsif ($command =~ /^dayendreport$/) { day_end_report($sqlrestriction); } elsif ($command =~ /^dailyjob$/) { portfolio_update($sqlrestriction); day_end_report($sqlrestriction); portfolio_status($sqlrestriction); portfolio_risk($sqlrestriction); } elsif ($command =~ /^delete$/) { delete_stock(@ARGV); } elsif ($command =~ /^destroydb$/) { deletedb(@ARGV); # does not return } elsif ($command =~ /^quote$/) { quote(@ARGV); } elsif ($command =~ /^retracement$/) { portfolio_retracement($sqlrestriction, "retrace"); } elsif ($command =~ /^risk$/) { portfolio_risk($sqlrestriction); } elsif ($command =~ /^status$/) { portfolio_status($sqlrestriction); } elsif ($command =~ /^update$/) { portfolio_update($sqlrestriction); } elsif ($command =~ /^lsportfolio$/) { lsportfolio($sqlrestriction); } elsif ($command =~ /^lspositions$/) { lspositions($sqlrestriction); } elsif ($command =~ /^deactivate$/) { inactive_stock(@ARGV); } elsif ($command =~ /^split$/) { split_stock(@ARGV); } elsif ($command =~ /^checkdbconnection$/) { CloseDB($dbh) or die("Cannot close db handle!\n"); print "Successfully connected to database from beancounter script.\n" if $Config{debug}; exit 1; } elsif ($command =~ /^warranty$/) { warranty(); } else { warn "Ignoring unknown command '$command'\n"; } } CloseDB($dbh); exit 0; # ---------------------- local functions ------------------------------------ sub help_and_exit { my $pmVersion = BeanCounterVersion; print STDERR " beancounter -- A stock portfolio performance monitoring tool beancounter version $version (RCS $rcsversion and $pmVersion) of $date Copyright (C) 1998 - 2006 by Dirk Eddelbuettel beancounter comes with ABSOLUTELY NO WARRANTY. This is free software, and you are welcome to redistribute it under certain conditions. Please try '$PROGRAM_NAME warranty' for more details, or visit the website at http://www.gnu.org/philosophy/free-sw.html Usage: beancounter [options] command [args] Commands: addindex index symbol1 [symbol2 [...]] add stock(s) to market index 'index' addportfolio symbol:nb:fx[:type:owner:price:date] [...] add n stock of s in currency fx to portfolio optional type, owner, purchase price and purch. date can also be given (see example) addstock symbol ... add stock(s) to database advancement report unrealized gains from lows allreports combines, dayendreport, status and risk backpopulate symbol ... fill with historic data for given stock(s) checkdbconnection test if connection to db can be established fxbackpopulate symbol ... fill with historic data for currency(ies) dailyjob combines update, dayendreport, status + risk dayendreport reports changes relative to the previous day deactivate symbol ... set stock(s) inactive in stockinfo table delete arg ... delete given stock(s) from database destroydb delete the BeanCounter database lsportfolio list portfolio data plreport p/l portfolio report against any other day retracement report unreal. losses from highs (drawdowns) quote arg ... report current data for given stock(s) risk display a portfolio risk report split arg ... split-adjust price history and portfolio status status report for a given date update update the database with current day's data warranty display short GNU GPL statement Options: --help show this help --verbose more verbose operation, mostly for debugging --date date use this as reference date for selected report --prevdate date use this as the previous reference date --currency fx use fx as the home currency --restriction sql impose this SQL restriction --extrafx fx1,fx2 additional currencies to load --forceupdate date force db to store new price info with date --[no]fxupdate enforce/suppress FX update [fxupdate] --[no]commit enforce/suppress database update [commit] --[no]equityupdate enforce/suppress Equity update, [update] --[no]ubcfx use/skip FX from UBC's Sauder school [skip] --splitby arg split stock history + position by factor [2] --rcfile file use different configuration file --dbsystem system use db backend system, default is PostgreSQL --dbname name use db name, default is beancounter Examples: beancounter update --forceupdate today beancounter addportfolio IBM:100:USD:401k:joe:121.4:20011205 VOD.L:50:GBP beancounter addstock CBOT LNUX RHAT COR.TO beancounter backpopulate MSFT IBM --prevdate 19940101 --date 19981231 beancounter fxbackpopulate EUR --prevdate 20010101 --date yesterday beancounter dayendreport --restriction \"type = '401k'\" beancounter status --date 20000816 --restriction \"currency='USD'\" beancounter split --splitby 3 --prevdate 1990-01-01 ABC CDE \n"; exit 1; } sub warranty { my $BeanCounterVersion = BeanCounterVersion; open (FILE, "< $PROGRAM_NAME"); my $over = 0; # have we already had comment lines? while () { # show header last if (m/\$Id/); # quit if we reach the RCS code next unless (m/^\#\s+/ or $over); $over = 1; # note the new state $ARG =~ s/^\#//; # minus the leading '#' print STDERR $ARG; } close(FILE); print STDERR " beancounter version $version ($BeanCounterVersion) as of $date\n\n"; } sub build_lines { my $len = shift; my $tl = "=" x $len; # thick line my $fl = "-" x $len; # fine line return ($tl,$fl); } sub display_report { my ($pretty_date, $pretty_prev_date, $prices, $prev_prices, $fx_prices, $prev_fx_prices, $shares, $fx, $pricedate) = @_; my ($tl,$fl) = build_lines(79); print "$tl\n" . "Profit / loss\t\t from $pretty_prev_date" . "\tto $pretty_date abs, rel change" . "\n$fl\n"; my (%value, %value_prev); foreach my $key (sort keys %$shares) { my ($name,$count) = split /:/, $key; if (Date_Cmp($pricedate->{$name}, $Config{lastbizday}) != 0) { $value{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; $value_prev{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}}/$fx_prices->{$Config{currency}}; } elsif ( defined($prev_prices->{$name}) ) { $value{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; $value_prev{$name} += $shares->{$key} * $prev_prices->{$name} * $prev_fx_prices->{$fx->{$name}}/$prev_fx_prices->{$Config{currency}}; } else { $value{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; $value_prev{$name} += $shares->{$key} * $prev_fx_prices->{$fx->{$name}}/$prev_fx_prices->{$Config{currency}}; } } my ($assets, $assets_prev) = (0,0); foreach my $name (sort keys %value) { my $value = $value{$name}; my $value_prev = $value_prev{$name}; print "$name $value_prev\n" if $Config{verbose}; if (Date_Cmp($pricedate->{$name}, $Config{lastbizday}) != 0) { printf("%*s %3s %19s %10.2f %8.2f\n", -18, substr($name,0,17), $fx->{$name}, " (from $pricedate->{$name})", $value, $prices->{$name}); } elsif ( ($value_prev == 0) || (!defined($prev_prices->{$name})) ) { printf("%*s %3s %20s %10.2f %8.2f %8.2f %7s\n", -18, substr($name,0,17), $Config{currency}, "(no price available)", $value, $prices->{$name}, $value-$value_prev, "N/A"); } else { printf("%*s %3s %10.2f %8.2f %10.2f %8.2f %8.2f %6.2f%%\n", -18, substr($name,0,17), $Config{currency}, $value_prev, $prev_prices->{$name}, $value, $prices->{$name}, $value-$value_prev, 100*($value/$value_prev-1)*Sign($value)); } $assets += $value; $assets_prev += $value_prev; } print "$fl\n"; printf("%-16s %3s %10.2f %10.2f %12.2f %6.2f%%\n", "Grand Total", $Config{currency}, $assets_prev, $assets, $assets-$assets_prev, 100*($assets/$assets_prev-1)); print "$tl\n"; } sub display_status { my ($date, $pretty_date, $prices, $pricedates, $fx_prices, $shares, $fx, $cost, $pdate, $cash) = @_; my ($tl,$fl) = build_lines(79); print "$tl\n" . "\t\t\tPortfolio Status on $pretty_date\n$fl\n"; printf("%*s %6s %14s %16s %11s %9s\n%s\n", -18, "Name", "Shares", "Close", "Position", "Held", "Return", $fl); my (%value, %totalshares); foreach my $key (sort keys %$shares) { my ($name,$count) = split /:/, $key; $value{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; $totalshares{$name} += $shares->{$key}; } my ($assets,$weightedreturn,$assetbase,$err) = (0,0,0,0); foreach my $name (sort keys %value) { my $value = $value{$name}; my ($daysheld,$return) = ("", ""); if (defined($pdate->{$name})) { # if we have a purchase date $daysheld = Delta_Format(DateCalc($pdate->{$name}, $date, \$err, 2), "approx", 0, "%dt"); if (defined($cost->{$name}) and $daysheld >= 0) { $return = ($prices->{$name} / $cost->{$name} - 1) * 100 * Sign($totalshares{$name}); # annualise if held longer than > 1 year $return *= 365 / $daysheld if ($daysheld > 365); # weigh returns by the invested amount, not the current value # NB this omits possible changes in the FX rate my $invested = $value / $prices->{$name} * $cost->{$name}; $weightedreturn += $return * $invested; $assetbase += $invested; } } printf("%*s %6d %3s %8.2f %3s %10.2f %s %s\n", -18, substr($name,0,17), $totalshares{$name}, $fx->{$name}, $prices->{$name}, $Config{currency}, $value, $daysheld ne "" ? sprintf("%5d days", $daysheld) : $daysheld, $return ne "" ? sprintf("%8.1f%%", $return) : $return); $assets += $value; } foreach my $name (sort keys %{$cash}) { # cash part my $value = $cash->{$name}{value} * $fx_prices->{$cash->{$name}{fx}} / $fx_prices->{$Config{currency}}; # adjust for FX $assets += $value; $assetbase += $value; printf("%*s %s %3s %10.2f\n", -16, substr($name,0,16), " " x 25, $Config{currency}, $value, ); } print "$fl\n"; printf("%-16s %29s %10.2f %20s\n", "Grand Total", $Config{currency}, $assets, $assetbase ? sprintf("%20.2f%%",$weightedreturn/$assetbase) : "NaN"); print "$fl\n"; print "Returns are annualized if the holding period exceeds one year.\n"; print "$tl\n"; } sub display_longstatus { my ($date, $pretty_date, $prices, $pricedates, $fx_prices, $shares, $fx, $cost, $pdate, $cash) = @_; my ($tl,$fl) = build_lines(79); print "$tl\n" . "\t\t\tPortfolio Status on $pretty_date\n$fl\n"; printf("%*s %6s %14s %16s %11s %9s\n%s\n", -18, "Name", "Shares", "Close", "Position", "Held", "Return", $fl); my (%value, %totalshares); foreach my $key (sort keys %$shares) { my ($name,$count) = split /:/, $key; $value{$key} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; } my ($assets,$weightedreturn,$assetbase,$err) = (0,0,0,0); foreach my $key (sort keys %value) { my ($name,$count) = split /:/, $key; my $value = $value{$key}; my ($daysheld,$return,$fxthen,$fxnow) = ("", "", 1.0, 1.0); if (defined($pdate->{$key})) { # if we have a purchase date $daysheld = Delta_Format(DateCalc($pdate->{$key}, $date, \$err, 2), "approx", 0, "%dt"); if (defined($cost->{$key}) and $daysheld >= 0 and $cost->{$key} != 0) { if ($fx->{$name} ne $Config{currency}) { $fxthen = GetFXDatum($dbh, $pdate->{$key}, $fx->{$name}); $fxnow = GetFXDatum($dbh, $date, $fx->{$name}); } $return = (($prices->{$name} * $fxnow) / ($cost->{$key} * $fxthen) - 1) * 100 * Sign($shares->{$key}); # annualise if held longer than > 1 year $return *= 365 / $daysheld if ($daysheld > 365); # weigh returns by the invested amount, not the current value # NB this omits possible changes in the FX rate my $invested = $value / $prices->{$name} * $cost->{$key}; $weightedreturn += $return * $invested; $assetbase += $invested; } } printf("%*s %6d %3s %8.2f %3s %10.2f %s %s\n", -18, substr($name,0,17), $shares->{$key}, $fx->{$name}, $prices->{$name}, $Config{currency}, $value, $daysheld ne "" ? sprintf("%5d days", $daysheld) : $daysheld, $return ne "" ? sprintf("%8.1f%%", $return) : $return); $assets += $value; } foreach my $name (sort keys %{$cash}) { # cash part my $value = $cash->{$name}{value} * $fx_prices->{$cash->{$name}{fx}} / $fx_prices->{$Config{currency}}; # adjust for FX $assets += $value; $assetbase += $value; printf("%*s %s %3s %10.2f\n", -16, substr($name,0,16), " " x 25, $Config{currency}, $value, ); } print "$fl\n"; printf("%-16s %29s %10.2f %20s\n", "Grand Total", $Config{currency}, $assets, $assetbase ? sprintf("%20.2f%%",$weightedreturn/$assetbase) : "NaN"); print "$fl\n"; print "Returns are annualized if the holding period exceeds one year.\n"; print "$tl\n"; } sub display_riskreport { my ($pretty_date, $pretty_prev_date, $var, $pos, $vol, $quintile, $fx, $crit, $margvar) = @_; my (%var); my ($varsum, $assets) = 0; foreach my $pkey (keys %$pos) { if (defined($pos->{$pkey}) && defined($vol->{$pkey})) { $var{$pkey} = $crit * sqrt($pos->{$pkey}**2 * $vol->{$pkey}**2); $varsum += $crit * sqrt($pos->{$pkey}**2 * $vol->{$pkey}**2); $assets += $pos->{$pkey}; } } my ($tl,$fl) = build_lines(79); print "$tl\n" . " Portfolio Risk on $pretty_date " . "going back to $pretty_prev_date\n$fl\n"; printf("%*s %15s %13s %10s %9s %7s\n%s\n", -18, "Name", "Position", "1% Profit/Loss", "Volatility", "VaR", "margVaR", $tl); foreach my $pkey (sort keys %$pos) { printf("%-18s %3s %11s %7s %7s %10s %9s %7d\n", substr($pkey,0,17), $Config{currency}, # NOT $fx->{$pkey}, !! defined($pos->{$pkey}) ? sprintf("%11.2f", $pos->{$pkey}) : 'N/A', defined($quintile->{$pkey}) ? sprintf("%6.1f%%", 100*$quintile->{$pkey}) : 'N/A', defined($quintile->{$pkey}) ? sprintf("%7d", $pos->{$pkey}*$quintile->{$pkey}) : 'N/A', defined($vol->{$pkey}) ? sprintf("%9.1f%%", 100*sqrt(252)*$vol->{$pkey}) : 'N/A', defined($var{$pkey}) ? sprintf("%8d", -($var{$pkey})) : 'N/A', $margvar->{$pkey}); } printf("$fl\n%-18s %3s %11.2f %24.1f%% %8d\n$fl\n", "Portfolio level", $Config{currency}, $assets, $var/$crit/$assets*100*sqrt(252), # back volatility out of VaR -$var); printf("%s is %4.1f%% of assets, or %5.1f%% of VaR sum of %13d\n%s\n", "Portfolio VaR", $var/$assets*100, $var/$varsum*100, -$varsum, $tl); print "VaR calculations use a 99% confidence level and 1-day horizon. Marginal VaR is\nthe change to the portfolio VaR attributable to adding this position. Computing\nthe 1% quintile of the return distribution, which can be viewed as a non-para-\nmetric VaR estimate, requires at least 100 observations.\n"; printf("$tl\n"); } sub split_stock { my @arg = @_; my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); print "Splitting stockprices by $splitby from $prev_date to $date:"; my $stmt = qq{select date, previous_close, day_open, day_low, day_high, day_close, bid, ask, volume from stockprices where symbol = ? and date <= ? and date >= ? and day_close > 0 order by date }; print "split_stock():\n\$stmt = $stmt\n" if $Config{debug}; my $sth = $dbh->prepare($stmt); foreach my $arg (sort @arg) { print " $arg "; my $rv = $sth->execute($arg, $date, $prev_date); my $dr = $sth->fetchall_arrayref; # get data $sth->finish; for (my $i=0; $i[$i][1]); $cmd.= " day_open = $dr->[$i][2]/$splitby," if defined($dr->[$i][2]); $cmd.= " day_low = $dr->[$i][3]/$splitby," if defined($dr->[$i][3]); $cmd.= " day_high = $dr->[$i][4]/$splitby," if defined($dr->[$i][4]); $cmd.= " day_close = $dr->[$i][5]/$splitby," if defined($dr->[$i][5]); $cmd.= " bid = $dr->[$i][6]/$splitby," if defined($dr->[$i][6]); $cmd.= " ask = $dr->[$i][7]/$splitby," if defined($dr->[$i][7]); $cmd.= " volume = $dr->[$i][8]*$splitby," if defined($dr->[$i][8]); $cmd.= "where symbol = '$arg' "; $cmd.= "and date = '$dr->[$i][0]'"; $cmd =~ s/,where/ where/; print "$cmd\n" if $Config{debug}; if ($Config{commit}) { $dbh->do($cmd) or warn "Failed for $arg with $cmd\n"; $dbh->commit(); } } } print " Done.\n"; print "Adjusting portfolio by $splitby:"; $stmt = qq{select shares, cost from portfolio where symbol = ? }; print "split_stock():\n\$stmt = $stmt\n" if $Config{debug}; $sth = $dbh->prepare($stmt); foreach my $arg (sort @arg) { print " $arg "; my $rv = $sth->execute($arg); my $dr = $sth->fetchall_arrayref; # get data $sth->finish; for (my $i=0; $i[$i][1]); $cmd.= "where symbol = '$arg' "; $cmd.= "and shares = '$dr->[$i][0]' "; $cmd.= "and cost = '$dr->[$i][1]' " if defined($dr->[$i][1]); $cmd =~ s/,where/ where/; print "$cmd\n" if $Config{debug}; if ($Config{commit}) { $dbh->do($cmd) or warn "Failed for $arg with $cmd\n"; $dbh->commit(); } } } print " Done.\n"; } sub lsportfolio { my @arg = @_; # statement for listing out portfolio my $stmt = qq{select * from portfolio }; # optional sql clauses for listing out portfolio if ($sqlrestriction) { $stmt = $stmt . qq{where $sqlrestriction }; } elsif ($fxarg) { $stmt = $stmt . qq{where currency='$fxarg' }; } elsif ($datearg ne "today") { $stmt = $stmt . qq{where date='$datearg' }; } else { # need a default clause $stmt = $stmt . qq{where date > '1970-01-01' }; } $stmt = $stmt . qq{and symbol in (select symbol from stockinfo where active) } . qq{order by date}; print "Portfolio listing sql statement:\n$stmt\n" if $Config{verbose}; my ($tl,$fl) = build_lines(79); # perform the access my $sth = $dbh->prepare($stmt); my $rows = $sth->execute; if ( defined($rows) and $rows > 0 ) { # print the results print "$tl\n\t\t\tPortfolio listing\n$fl\n"; printf "%-10.10s%8.8s%8.8s%12.12s%12.12s%12.12s%17.17s\n$fl\n", "Symbol", "Shares", "FX", " Type", " Owner", " Cost", " Date"; while (my @row_ary = $sth->fetchrow_array) { my ($stock, $nb, $fx, $type, $owner, $cost, $date) = @row_ary; printf "%-10.10s%8d%8s%12s%12s%12.2f%17s\n", $stock, $nb, $fx, $type, $owner, $cost, $date; } print "$tl\n"; } $sth->finish; } sub lspositions { my $res = shift; my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); my ($fx, $prices, $prev_prices, $shares, $pricedate) = GetPriceData($dbh,$date,$res); $fx->{'home currency'} = $Config{currency}; my ($tl,$fl) = build_lines(79); # statement for listing out portfolio my $stmt = qq{select p.symbol, p.shares, p.currency, p.type, p.owner, } . qq{p.cost, p.date, s.name from portfolio p, stockinfo s }; # optional sql clauses for listing out portfolio if ($sqlrestriction) { $stmt = $stmt . qq{where $sqlrestriction }; } else { # need a default clause $stmt = $stmt . qq{where p.date > '1970-01-01' }; } if ($fxarg) { $stmt = $stmt . qq{and p.currency='$fxarg' }; } $stmt = $stmt . qq{and s.active } . qq{and p.symbol = s.symbol } . qq{order by p.date}; print "Portfolio listing sql statement:\n$stmt\n" if $Config{verbose}; # perform the access my $sth = $dbh->prepare($stmt); my $rows = $sth->execute; if ( defined($rows) and $rows > 0 ) { # print the results print "$tl\n\t\t\t\tPortfolio listing\n$fl\n"; printf "%-7.7s%7.7s%5.5s%8.8s%8.8s%8.8s%12.12s%8.8s%11s\n$fl\n", "Symbol", "Shares", "FX", " Type", " Owner", " Cost", " Date", " Price", " Net"; my $sum = 0; while (my @row_ary = $sth->fetchrow_array) { my ($fxthen,$fxnow) = (1.0,1.0); my ($stock, $nb, $fx, $type, $owner, $cost, $pfdate, $name) = @row_ary; if ($fx ne $Config{currency}) { $fxthen = GetFXDatum($dbh, $pfdate, $fx); $fxnow = GetFXDatum($dbh, $date, $fx); print "Running SQL for $pfdate and $fx: $fxthen -> $fxnow\n" if $Config{debug}; } my $pnl = $nb * ($prices->{$name}*$fxnow - $cost*$fxthen); printf "%-7.7s%7d%5s%8s%8s%8.2f%12s%8.2f%11.2f\n", $stock, $nb, $fx, $type, $owner, $cost, $pfdate, $prices->{$name}, $pnl; $sum = $sum + $pnl; } printf "$fl\nTotal profit or loss\t\t\t\t\t%18.2f\n$tl\n", $sum; } $sth->finish; } sub day_end_report { my $res = shift; # day-end report is always relative to the previous day, we enforce previous # date of 24 hrs before given date by unsetting any cmdline argument that # there might have been as the function will then return previous biz day my $prevdatesaved = $prevdatearg; $prevdatearg = undef; my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); my ($fx, $prices, $prev_prices, $shares, $pricedate) = GetPriceData($dbh,$date,$res); $fx->{'home currency'} = $Config{currency}; my ($fx_prices) = GetFXData($dbh, $date, $fx); my ($prev_fx_prices) = GetFXData($dbh, $prev_date, $fx); display_report($pretty_date, $pretty_prev_date, $prices, $prev_prices, $fx_prices, $prev_fx_prices, $shares, $fx, $pricedate); $prevdatearg = $prevdatesaved; # and reset any potential cmdline arg } sub portfolio_retracement { my ($res, $mode) = @_; ## need to do a sanity checks on the date die "Error: '$prevdatearg' not prior to '$datearg'.\n" unless (Date_Cmp(ParseDate($prevdatearg), ParseDate($datearg)) < 0); my $err = 0; die "Error: '$prevdatearg' is not at least 30 days prior to '$datearg'.\n" unless (Delta_Format(DateCalc(ParseDate($prevdatearg), ParseDate($datearg), \$err, 2), "approx", 0, "%dt") > 30); my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); my ($fx, $prices, $tmp1, $shares, $pricedate) = GetPriceData($dbh,$date,$res); $fx->{'home currency'} = $Config{currency}; my ($fx_prices) = GetFXData($dbh, $date, $fx); my ($highprev, $lowprev) = GetRetracementData($dbh, $date, $prev_date, $res, $fx_prices); my (%value, %totalshares); foreach my $key (sort keys %$shares) { my ($name,$count) = split /:/, $key; $value{$name} += $shares->{$key} * $prices->{$name} * $fx_prices->{$fx->{$name}} / $fx_prices->{$Config{currency}}; $totalshares{$name} += $shares->{$key}; } my $text = ($mode eq "advance") ? "Advances" : "Retracement"; my ($tl,$fl) = build_lines(79); my ($totaldown, $weightedup, $assetbase) = (0,0,0); print "$tl\n"; printf("%-17s", $text); print " shares price value $prev_date to $date unreal. "; print "" . ($mode eq "advance") ? "gain" : "loss"; print "\n$fl\n"; foreach (sort keys %$prices) { my ($downperc,$downamount,$refprice); $refprice = $highprev->{$ARG}; # compare to recent high, or low for short if (($mode eq "retrace" and $totalshares{$ARG} < 0) or # or inverse of it ($mode eq "advance" and $totalshares{$ARG} > 0)) { # if advance $refprice = $lowprev->{$ARG}; } $downperc = $prices->{$ARG}/$refprice - 1; $downamount = $totalshares{$ARG} * ($prices->{$ARG} - $refprice) * $fx_prices->{$fx->{$ARG}} / $fx_prices->{$Config{currency}}; printf("%-17s %6d %8.2f %10.2f %8.2f %10.2f%% %12.2f\n", substr($ARG, 0, 17), $totalshares{$ARG}, $prices->{$ARG}, $value{$ARG}, $refprice, 100*$downperc, $downamount); $totaldown += $downamount; $assetbase += $value{$ARG}; } print "$fl\n"; printf("%-29s %10.2f %8.2f%% %13.2f\n", "Aggregate", $assetbase, 100*$totaldown/$assetbase, $totaldown); print "$tl\n"; } sub portfolio_risk { my $res = shift; my $crit = 2.326348; # 1% critical value of the Normal distribution #my $crit = 1.644584; # 5% critical value of the Normal distribution ## need to do a sanity checks on the date die "Error: '$prevdatearg' not prior to '$datearg'.\n" unless (Date_Cmp(ParseDate($prevdatearg), ParseDate($datearg)) < 0); my $err = 0; die "Error: '$prevdatearg' is not at least 30 days prior to '$datearg'.\n" unless (Delta_Format(DateCalc(ParseDate($prevdatearg), ParseDate($datearg), \$err, 2), "approx", 0, "%dt") > 30); my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); my ($fx, $prices, $tmp1, $shares, $pricedate) = GetPriceData($dbh,$date,$res); $fx->{'home currency'} = $Config{currency}; my ($fx_prices) = GetFXData($dbh, $date, $fx); my ($var, $pos, $vol, $quintile, $margvar) = GetRiskData($dbh, $date, $prev_date, $res, $fx_prices, $crit); display_riskreport($pretty_date, $pretty_prev_date, $var, $pos, $vol, $quintile, $fx, $crit, $margvar); } sub portfolio_status { my $res = shift; my ($date, $prev_date) = GetTodaysAndPreviousDates(); # override with optional dates, if supplied $date = UnixDate(ParseDate($datearg), "%Y%m%d") if ($datearg); # create 'prettier' non-ISO 8601 form my $pretty_date = UnixDate(ParseDate($date), "%d %b %Y"); my ($fx, $prices, $prev_prices, $shares, $pricedates, $cost, $pdate) = GetPriceData($dbh,$date,$res); $fx->{'home currency'} = $Config{currency}; my ($fx_prices) = GetFXData($dbh, $date, $fx); my $cash = GetCashData($dbh, $date, $res); display_longstatus($date, $pretty_date, $prices, $pricedates, $fx_prices, $shares, $fx, $cost, $pdate, $cash); } # portfolio_report -- with two "free" dates, ie from last month to last week sub portfolio_report { my $res = shift; ## need to do a sanity check on the date unless (Date_Cmp(ParseDate($prevdatearg), ParseDate($datearg)) < 0) { warn "Error: Date $prevdatearg not prior to date $datearg\n"; } my ($date, $prev_date, $pretty_date, $pretty_prev_date) = GetTodaysAndPreviousDates(); my ($fx, $prices, $tmp1, $shares, $pricedate) = GetPriceData($dbh,$date,$res); my ($tmp2, $prev_prices, $tmp3, $tmp4, $prev_pricedate) = GetPriceData($dbh,$prev_date,$res); $fx->{'home currency'} = $Config{currency}; my ($fx_prices) = GetFXData($dbh, $date, $fx); my ($prev_fx_prices) = GetFXData($dbh, $prev_date, $fx); display_report($pretty_date, $pretty_prev_date, $prices, $prev_prices, $fx_prices, $prev_fx_prices, $shares, $fx, $pricedate, $prev_pricedate); } sub portfolio_update { my $res = shift; UpdateFXviaUBC($dbh, $res) if $Config{fxupdate} and $Config{ubcfx}; UpdateFXDatabase($dbh, $res) if $Config{fxupdate}; # update FX db UpdateDatabase($dbh, $res) if $Config{equityupdate}; } sub backpopulate { my @arg = @_; my $fromdate = 19990101; # default to start in Jan of 1999 my $todate = UnixDate(ParseDate("yesterday"),"%Y%m%d"); $fromdate = UnixDate(ParseDate($prevdatearg),"%Y%m%d") if ($prevdatearg); $todate = UnixDate(ParseDate($datearg),"%Y%m%d") if ($datearg); my $stmt = qq{ select symbol from stockinfo where symbol = ? }; my $sth; if (@arg) { $sth = $dbh->prepare($stmt); } foreach my $symbol (@arg) { # we need to query the DB to see if this symbol was registered my $rv = $sth->execute(uc $symbol); # Yahoo only returns .csv for non-us stocks in 200 data point lots my @startdates; my @enddates; # build an array pair of start and endates which increment from # $fromdate to $todate in about 190 business day steps my $yahoostartdate = $fromdate; my $yahooenddate = substr DateCalc($yahoostartdate, "+190 business days"),0,8; # enter/continue this loop if end date is still further # out than $yahooenddate while ($yahooenddate < $todate) { print "Date Pair: $yahoostartdate to $yahooenddate\n" if $Config{verbose}; push @startdates,$yahoostartdate; push @enddates,$yahooenddate; # move the start date of the next pair to 1 past end of the last pair $yahoostartdate = substr DateCalc($yahooenddate, "+1 business days"),0,8; # calculate the next end date - add 190 business days $yahooenddate = substr DateCalc($yahoostartdate, "+190 business days"),0,8; } # get the last date pair which will end precisely on $todate if ($yahooenddate >= $todate) { # this date range does not require any breaking up print "Final Date Pair: $yahoostartdate to $todate\n" if $Config{verbose}; push @startdates,$yahoostartdate; push @enddates,$todate; } # write all the stocks we find into the db while (my $r = $sth->fetch) { # roughly, if a symbol has a non-us exchange ie it has a dot in # the symbol name then yahoo will only give us 200 data points at # one go so we need to loop through the various dates if ($r->[0]=~/.+\..+/) { # loop through the startdates/enddates array pairs getting #the data as we go while ($yahoostartdate=pop @startdates) { # get the matching end date as well $yahooenddate=pop @enddates; print " Adding $r->[0] from $yahooenddate to $yahoostartdate\n"; my @arr = GetHistoricalData($r->[0], $yahoostartdate, $yahooenddate); PrintHistoricalData($r->[0], @arr) if $Config{verbose}; DatabaseHistoricalData($dbh, $r->[0], @arr); } } else { print " Adding $r->[0] from $fromdate to $todate\n"; my @arr = GetHistoricalData($r->[0], $fromdate, $todate); PrintHistoricalData($r->[0], @arr) if $Config{verbose}; DatabaseHistoricalData($dbh, $r->[0], @arr); } } $sth->finish; } } sub fxbackpopulate { my @arg = @_; my $fromdate = 20000101; # default to start in Jan of 2000 my $todate = UnixDate(ParseDate("yesterday"),"%Y%m%d"); $fromdate = UnixDate(ParseDate($prevdatearg),"%Y%m%d") if ($prevdatearg); $todate = UnixDate(ParseDate($datearg),"%Y%m%d") if ($datearg); if ($Config{ubcfx}) { # if we use PACIFIC at UBC's Sauder School my $aref = GetUBCFXData(\@arg, $fromdate, $todate); #print " backpopulating ", join(" ",@arg), # " from $fromdate to $todate\n" if $Config{debug}; #no equiv. here: #PrintHistoricalData($iso, @arr) if $Config{verbose}; DatabaseHistoricalUBCFX($dbh, $aref, @arg); ##DatabaseHistoricalFXData($dbh, GetYahooCurrency($iso), @arr); } else { foreach my $iso (@arg) { # my $fx = GetYahooCurrency($iso); # if (defined($fx) and $fx ne "") { print " backpopulating $iso (rel. to $Config{currency}) ". "from $fromdate to $todate\n"; my $aref = GetOandAFXData($iso, $fromdate, $todate); PrintHistoricalData($iso, @{$aref}) if $Config{verbose}; DatabaseHistoricalOandAFX($dbh, $aref, @arg); # #if $Config{debug}; # my (@arr) = GetHistoricalData($fx, $fromdate, $todate); # PrintHistoricalData($fx, @arr) if $Config{verbose}; # DatabaseHistoricalFXData($dbh, $fx, @arr); # } else { # print "** Ignoring $iso which is not currently known to " . # "the internal tables\n"; # } } } } sub quote { my @arg = @_; my @data = GetDailyData(@arg); # fetch data my %data = ParseDailyData(@data); # fill assoc. array ReportDailyData(%data); # report data } sub add_index { my @arg = @_; my $index = shift @arg; # get the index argument foreach my $arg (@arg) { # and loop over the stocks my $cmd = "insert into indices values ( '$arg', '$index' );"; print "$cmd\n" if $Config{verbose}; $dbh->do($cmd) or warn "\nFailed with $cmd\n"; $dbh->commit(); # the ODBC driver needs that for a weird reason } } sub add_stock { my @arg = @_; my @data = GetDailyData(@arg); my %data = ParseDailyData(@data); ReportDailyData(%data) if $Config{verbose}; DatabaseInfoData($dbh, %data); DatabaseDailyData($dbh, %data); } sub add_portfolio { my @arg = @_; # statement for insertion into portfolio my $stmt = qq{insert into portfolio values ( ?, ?, ?, ?, ?, ?, ? );}; my $sth; # statement to check if this symbol already in info my $infostmt = qq{ select symbol from stockinfo where symbol = ? }; my $infosth; if (@arg) { $infosth = $dbh->prepare($infostmt); } my @symbol; foreach my $arg (@arg) { # and loop over the stocks my ($stock,$nb,$fx,$type,$owner,$cost,$date) = (undef, undef, undef, undef, undef, undef, undef); print "Inserting $arg\n" if $Config{verbose}; ($stock,$nb,$fx,$type,$owner,$cost,$date) = split /:/, $arg; $stock = uc $stock; # uppercase it $fx = $Config{currency} unless defined($fx); if (defined($stock) and defined($nb)) { if (!defined($sth)) { $sth = $dbh->prepare($stmt); } $sth->execute($stock, $nb, $fx, $type, $owner, $cost, $date); $sth->finish; $dbh->commit(); # the ODBC driver needs that for a weird reason } else { warn "Ignoring invalid argument $arg\n"; } $infosth->execute($stock); push @symbol, $stock unless ($infosth->fetch); $infosth->finish; } if ($#symbol > -1) { # if there are symbols to be added add_stock(@symbol) # make sure 'new' stocks get added to DB } } # set 'active' field false in stockinfo table for 'symbol' # no sql restrictions are applicable to the stockinfo table sub inactive_stock { my @arg = @_; # array of stock symbols # statement to check if this symbol already in info my $infostmt = qq{select symbol,active from stockinfo where symbol=?;}; my $infosth = $dbh->prepare($infostmt); printf STDERR "infostmt:%s\n", $infostmt if $Config{verbose}; ## for all arguments ## if stock is 1) in info table and 2) active then deactive it ## else report condition a) not in table or b) not active my @symbol; foreach my $arg (@arg) { # and loop over the stocks my $stock = uc $arg; # uppercase it print "Checking stock $stock\n" if $Config{verbose}; $infosth->execute($stock); my ($symb, $active) = $infosth->fetchrow_array; $infosth->finish; if (! $symb) { printf STDERR "Symbol %s not in database, cannot deactive\n", $stock; } else { if ($active == 1) { push @symbol, $symb; } else { printf STDERR "Symbol %s is already deactivated\n", $symb; } } } # statement for updating stockinfo table my $stmt = qq{update stockinfo set active='false' where symbol=?;}; my $sth = $dbh->prepare($stmt); print STDERR "inactive stock sql statement:\n$stmt\n" if $Config{verbose}; if ($Config{commit}) { ## with $stock(s) in @symbol do the deed foreach my $stk (@symbol) { print STDERR "Inactivating stock $stk\n" if $Config{verbose}; # perform the access $sth->execute($stk); $sth->finish; $dbh->commit(); # the ODBC driver needs that for a weird reason } } } sub delete_stock { my @arg = @_; foreach my $arg (@arg) { # and loop over the stocks foreach my $table (qw/ stockinfo stockprices indices portfolio/) { my $cmd = "delete from $table where symbol = '$arg';"; print "$cmd\n" if $Config{verbose}; $dbh->do($cmd) or warn "\nFailed with $cmd\n"; $dbh->commit(); # the ODBC driver needs that for a weird reason } } } sub deletedb { # this unfortunately is PostgreSQL specific CloseDB($dbh); # or we won't be able to close if (lc $Config{dbsystem} eq "mysql") { system("mysqladmin drop $Config{dbname}") or warn("Could not delete MySQL database $Config{dbname}"); ## bad return value, will trigger die() even on success } elsif (lc $Config{dbsystem} eq "postgresql") { # system("psql -q -c \"drop database beancounter\"") == 0 system("dropdb $Config{dbname}") # requires PostgreSQL >= 7.1 or warn("Could not delete Postgresql database $Config{dbname}"); ## bad return value, will trigger die() even on success } elsif (lc $Config{dbsystem} eq "sqlite" or # sqlite v3 lc $Config{dbsystem} eq "sqlite2") { # sqlite v2 unlink($Config{dbname}) or warn("Could not remove SQLite database $Config{dbname}"); } exit(0); } __END__ # that's it, folks! Documentation below #---- Documentation --------------------------------------------------------- =head1 NAME beancounter - Stock portfolio performance monitor tool =head1 SYNOPSYS beancounter [options] command [command_arguments ...] =head1 COMMANDS addindex index args add stock(s) to market index 'indx' addportfolio sym:nb:fx:type:o:pp:pd ... add 'nb' stocks of company with symbol 'sym' that are listed in currency 'fx' to the portfolio with optional 'type' and 'owner' info, purchase price 'pp' and date 'pd'; see below for a complete example allreports combines dayendreport, status and risk addstock arg ... add stock(s) with symbol arg to the database advancement report on unrealized gains from lows backpopulate arg ... fill with historic data for given stock(s) checkdbconnection test if connection to db can be established dailyjob combines update, dayendreport, status + risk dayendreport reports p/l changes relative to previous day deactivate symbol ... set stock(s) inactive in stockinfo table delete arg ... delete given stock(s) from database destroydb delete the BeanCounter database fxbackpopulate arg ... fill with historic data for currency(ies) lsportfolio list portfolio data plreport run an portfolio p/l report rel. to any day quote arg ... report current data for given stock(s) retracement report unrealized losses from highs (drawdowns) risk display a portfolio risk report split arg ... split-adjust price history and portfolio status status summary report for portfolio update update the database with day's data warranty display the short GNU GPL statement =head1 OPTIONS --help show this help --verbose more verbose operation, debugging --date date report for this date (today) --prevdate date relative to this date (yesterday) --currency fx set home currency --restriction sql impose SQL restriction --extrafx fx1,fx2,... additional currencies to load --forceupdate date force db to store new price info with date --rcfile file use different configuration file --[no]fxupdate enforce/suppress FX update, default is update --[no]commit enforce/suppress database update, default is commit --[no]equityupdate enforce/suppress Equity update, default is update --[no]ubcfx use/skip FX from UBC's Sauder school, default skip --splitby arg split stock history + position by this factor [2] --dbsystem system use db backend system, default is PostgreSQL --dbname name use db name, default is beancounter =head1 DESCRIPTION B gathers and analyses stock market data to evaluate portfolio performance. It has several modes of operation. The first main mode is data gathering: both current data (e.g. end-of-day closing prices) and historical price data (to back-populate the database) can be retrieved both automatically and efficiently with subsequent local storage in a relational database system (either F, F or F) though any other system with an F driver could be used). The second main mode is data analysis where the stored data is evaluated to provide performance information. Several canned reports types are already available. Data is retrieved very efficiently in a single batch query per Yahoo! host from the Yahoo! Finance web sites using Finance::YahooQuote module (where version 0.18 or newer is required for proxy support). Support exists for North America (i.e. US and Canada), Europe (i.e. the Continent as well as Great Britain), several Asian stock markets, Australia and New Zealand. B can aggregate the change in value for the entire portfolio over arbitrary time horizons (provided historical data has either been gathered or has been backpopulated). Using the powerful date-parsing routine available to Perl (thanks to the F modules), you can simply say 'from six months ago to today' (see below for examples). B has been written and tested under Linux. It should run under any standard Unix as long as the required Perl modules are installed, as as long as the database backend is found. =head1 EXAMPLES beancounter update --forceupdate today This updates the database: it extends timeseries data (such as open, low, high, close, volume) with data for the current day, and overwrites static data (such as capital, price/earnings, ...) with current data. All stocks held in the database are updated (unless the --restriction argument instructs otherwise). The --forceupdate option lets the program corrects incorrect dates returned from Yahoo! (which happens every now and so often), but be careful to correct for this on public holidays. Note that the --restriction argument will be applied to the portfolio table, whereas the overall selection comes from the stockinfo table. beancounter addportfolio IBM:100:USD:401k:joe:90.25:20000320 \ SPY:50:USD:ira:joe:142.25:20000620 This adds IBM to the 401k portfolio of Joe, as well as SP500 'Spiders' to his IRA portfolio. The stocks are also added to the general stock info tables via an implicit call of the stockinfo command. beancounter addstock LNUX RHAT COR.TO This adds these three Linux companies to the database without adding them to any specific portfolios. beancounter backpopulate --prevdate '1 year ago' \ --date 'friday 1 week ago' IBM MSFT HWP This backpopulates the database with historic prices for three hardware companies. Note how the date specification is very general thanks to the underlying Date::Manip module. beancounter fxbackpopulate --prevdate '1 year ago' \ --date 'friday 1 week ago' CAD EUR GBP This backpopulates the database with historic prices for these three currencies. Note how the date specification is very general thanks to the underlying Date::Manip module. Unfortunately, Yahoo! is a little bone-headed in its implementation of historic FX rates -- these are stored to only two decimals precision, just like stockprices. Unfortunately, convention is to use at least four if not six. Because of the limited information, risk from FX changes will be underestimated. beancounter plreport --prevdate '1 month ago' --date 'today' \ --restriction "owner='joe'" This calculates portfolio profits or losses over the last month. It also imposes the database restriction that only stocks owned by 'joe' are to be included. beancounter status --restriction "type='401k'" This shows a portfolio status report with the restriction that only stocks from the '401k' account are to be included. beancounter risk --prevdate "6 month ago" This shows a portfolio risk report. This tries describes the statistically plausible loss which should be exceeded only 1 out of 100 times (see below for more details). beancounter dailyjob --forceupdate today Run a complete 'job': update the database, show a day-end profit/loss report, show a portfolio status report and show a riskreport. In the update mode, override a potentially wrong date supplied by Yahoo! with the current date. beancounter split --splitby 3 --prevdate 1990-01-01 ABC CDE Split-adjusts the (hypothetical) stocks ABC and CDE by a factor of three: price data in the database is divided by three, volume increased by 3 and similarly, in the portfolio shares are increased and cost is descreased. Default dates are --prevdate and --date which may need adjusting. =head1 TUTORIAL The following few paragraphs will illustrate the use of B. We will set up two fictional accounts for two brothers Bob and Bill (so that we can illustrate the 'owner' column). The prices below are completely fictitious, as are the portfolios. We suppose that B is installed and that the B command has been run. We can then create a two-stock (computer hardware) portfolio for Bob as follows: beancounter addportfolio SPY:50:USD:401k:bob:142.25:20000620 \ IBM:100:USD:401k:bob:90.25:20000320 Here we specify that 100 shares each of SPY and IBM, priced in US Dollars, are in Bob's portfolio which is tagged as a 401k retirement account. The (fictitious) purchase price and date are also given. Let's suppose that Bill prefers networking equipment, and that he has a brokerage account in Canada: beancounter addportfolio CSCO:100:USD:spec:bill:78.00:19990817 \ NT.TO:200:CAD:spec:bill:cad:90.25:20000212 Now we can backpopulate the database from 1998 onwards for all four stocks: beancounter backpopulate --prevdate 19980101 CSCO IBM NT.TO SPY With this historical data in place, we now compare how Bob's portfolio would have fared over the last 18 months: beancounter plreport --prevdate '18 months ago' \ --restriction "owner='bob'" Note how we use double quotes to protect the arguments, and how the SQL restriction contains a further single quote around the literal string. We can also review the performance for Bill at the most recent trading day: beancounter dayendreport --restriction "owner='bill'" or the status of holdings and their respective values: beancounter dayendreport --restriction "owner='bill'" Similarly, a risk reports can be run on this portfolios per beancounter risk --restriction "owner='bill'" =head1 MORE DETAILED COMMAND DESCRIPTION B is the most important 'position entry' command. As with other commands, several arguments can be given at the same time. For each of these, records are separated using a colon and specify, in order, stock symbol, number of stocks held, currency, account type, account owner, purchase price and purchase date. Only the first three arguments are required, the others are optional. Executing B implicitly executes B. The account type column can be used to specify whether the account is, e.g., a tax-sheltered retirement account, or it could be used to denote the brokerage company is it held at. B retrieves the most recent quotes(s). This is useful for illiquid securities which might not have traded that day, or if a public holiday occurred, or if there was a data error at Yahoo!. Two dates can be specified which determine the period over which the profit or loss is computed. This will fail if price data (or currency data in the case of foreign stocks data) data is not available for either of those two dates. This may be restrictive for foreign stocks where we cannot backpopulate due to lack of public data source for historical currency quotes. Major currencies can be retrieved from Yahoo!, but only to two decimals precisions. B is similar to B but is always over a one-day period. It also uses only one date record by calculating performance given the 'previous close' data. B shows holdings amounts, total position values, annualized returns in percentages and holding periods in days. Note that the annualized returns can appear excessive if, e.g., a ten-day return from a recently purchased stock is extrapolated to an annual time period. B shows a portfolio risk report which describes the statistically plausible loss which should be exceeded only 1 out of 100 times. In other words, the loss estimate has a critical level of 99%. This risk level is estimated via two methods. The first is non-parametric and assumes no particular model or distribution; it computes the 1% quintile of the return distribution and displays it as well as the corresponding asset value at risk. The second method uses the standard Value-at-Risk (VaR) approach. This uses the 1% critical value of the Normal distribution and implicitly assumes a normal distribution for returns. See C for more introduction and references. If the distribution of normalitty was perfectly true, both measures would coincide. A large difference between the two estimates would indicate that the return distribution might be rather non-normal. Another view of the riskiness of a given position is provided by the last column with the 'margVaR' heading. It shows the marginal Value-at-Risk. Marginal VaR is commonly defined as the risk contribution of the given position to the total portfolio, and calculated as the difference in the VaR of the full portfolio and the VaR of an otherwise identical portfolio with the given position removed. Note that calculating marginal VaR is fairly slow (on the order of O(n^3) ]. B shows a 'drawdown' report. Drawdown is commonly defined as the percentage loss relative to the previous high. The default period is used, but can be altered with the B<--date> and B<--prevdate> options. The default period is also corrected for the actual holding period. In other words, if a stock has been held for two months, only those two months are used instead of the default of six months -- but if the last months has been selected via B<--prevdate> then it is used. For short positions, the analysis is inverted and relative to the previous low. The report displays each stock, the number of shares held, the current price and holdings value. The next two columns show the maximum price attained in the examined period, and the percent decline relative to it. The last column shows the unrealized loss relative to the maximum price over the period. The aggregate holdings value, percent decline and unrealized loss are shown as well. B does the opposite of drawdown -- it computes unrealized gains relative to the minimum price in the period. The discussion in the preceding paragraph applies `but inverted'. B simply lists the content of the portfolio table. A SQL restriction can be imposed. B adds stocks a the index table. Currently, no further analysis references this table. B adds stocks to the database. From then on data will be retrieved for the given symbol(s) and stored in the database whenever the B command is executed. B fills the database with historic prices for the given symbols and date period. Note that this works well for stocks and mutual fund. Options have no historic data stored. Currencies are stored with limited precision as noted above. B simply shows a price quote for the given symbol(s). B updates the database with quotes for all stocks for the given day. No output is generated making the command suitable for B execution. B is a simple convenience wrapper around B, B, B and B, B is a another covenience wrapper around B, B and B. B will set the active column in stockinfo for the given symbol(s) to false thereby inhibiting any further updates of symbol(s). The existing data for symbol(s) is retained. Use this when a stock is acquired, delisted, or you simply want to stop tracking it -- but do not want to purge the historical data. B adjusts the price database, and the portfolio holdings, for stock splits. The default factor is 2, this can be adjusted with the option B<--splitby>. The dates arguments can be set with B<--prevdate> and B<--date>. B removes the given symbols from the database. B deletes the BeanCounter database. B simply opens and closes the database handle, and returns a specified exit code which can then be tested. This is used in the B command. B display a short GNU General Public License statement. =head1 MORE DETAILED OPTION DESCRIPTION B<--currency> can be used to select a different I currency. Instead of having all values converted to the default currency, the selected currency is used. B<--date> allows to choose a different reference date. This is then be be used by commands working on a date, or date period, such as B, B, B, B or B. B<--prevdate> allows to choose a different start date for return calculations, or data gathering. B<--restriction> can be used to restrict the database selection. The argument must be a valid part of valid SQL statement in the sense that existing columns and operators have to be employed. The argument to this option will be completed with a leading I. The SQL restriction will typcally be over elements of the I table which comprises the columns I, I, I, I, I, I and I. A simple example would be I. Note that this has to protected by double quotes I<"I> on the command-line. B<--extrafx> allows to gather data on additional currency rates beyond those automatically selected as shares are listed in them. A typical example would be for a European investor wanting to convert from the EUR in which the shares are listed into one of the member currencies which B would no longer retrieve as shares are no longer listed in these. B<--forceupdate> allows to overwrite an potentially wrong date in the database update. Unfortunately, it appears that Yahoo! occasionally reports correct prices with an incorrect date such as the previous day's. In such a case, this option, along with an argument such as 'today' can override the bad date datapoint and avoid a hole in the database. The downside of this approach is that it would "double" the previous data in the case of a public holiday, or even if it was run the weekend. A somewhat smarter comparison to previously stored data might prevent that, but would be more complex to implement. B<--rcfile> allows to specify a resource file different from the default I<~/.beancounterrc>. B<--dbsystem> allows to switch to a different database backend. The default is B but B and B are also supported. For B, the default is now version 3.* but the previous version -- which is not binarily compatible -- is supported as well with argument 'SQLite2'. B<--dbname> allows to switch to an alternate database. The default is 'beancounter'. This can be useful for testing new features. B<--fxupdate> is a boolean switch to enforece updates of FX rates during 'update'. The default is 'true' but '--nofxupdate' can be used to suppress the update of foreign exchange rates. Similarly, B<--equityupdate> is a boolean switch to enforece, or suppress updates of Equity (i.e. stock) data during 'update'. The default is 'true' but '--noequityupdate' can be used to suppress the update of foreign exchange rates. B<--ubcfx> is a boolean switch to use the 'PACIFIC' FX rate service from the Sauder School at UBC. This is useful when the default FX rate service at Yahoo! is erratic, or unreliable. While the PACIFIC server provides a wider variety of exchange rates, Yahoo! can still be useful as it can provide more columns (open/high/low). However, during most of 2005, Yahoo! has been unrealiable for the exchange rates and has not provided historical FX data. On the other hand, the UBC service does not run on Canadian holidays so it cannot really server as a full substitute. Contributions for a new data acquisition, maybe via www.oanda.com would be welcome. B<--splitby> can be used to set a stock split factor other than the default of 2. B<--host> can be used to point to a machine containing the PostgreSQL or MySQL database. The machine can be remote, or it can be the actual machine B is running on. If a hostname is given, tcp/ip connection are used. If no hostname is given, the default value of 'localhost' implies that local socket connections are used which may be easier to employ for less experienced adatabase users. Also, B<--commit> is a boolean switch to suppress actual database updates if the negated B<--nocommit> is selected. This is useful mostly in debugging contexts. The B<--verbose> and B<--debug> switches can be used in debugging an testing, and B<--help> triggers the display of help message. =head1 SYSTEM OVERVIEW The following section details some of the database and configuration options. =head2 DATABASE REQUIREMENTS B currently depends on either PostgreSQL, MySQL, SQLite (version 2 or 3) or any other database for which an ODBC driver is available (though the required tables would have to created manually in the ODBC case). Yet another DB backend could be added provided suitable Perl DBI drivers are available. For PostgreSQL, MySQL and SQLite, the B script can create and initialize the database, form the required tables and fills them with some example data. It is a starting point for local modifications. The connection to the database is made via a dedicated function in the B module, changes would only have to be made there. As of this writing the B (the database-independent interface for Perl) is used along the DBI drivers for PostgreSQL, MySQL, SQLite and ODBC. Ports for Oracle, Sybase, ... are encouraged. =head2 CONFIG FILE A configuration file F<~/.beancounterrc> is read if found. It currently supports the following options: =over 8 =item I to specify into which home currency holdings and profits/losses have to be converted =item I to specify the database server on which the B database resides (this is needed only for the alternate connection via the DBI-Pg driver in case DBI-ODBC is not used) =item I to specify the userid for the database connection; if needed. If not specified, the current user id is used. =item I to specify the password for the database connection, if needed. =item I to select a database backend, e.g. to switch from PostgreSQL to MySQL or SQLite or SQLite2 (the previous format of SQLite). =item I to select a different default database name other than the default of 'beancounter' =item I to specify the address of a firewall proxy server if one is needed to connect to the Internet. =item I to specify a firewallid:firewallpasswd combination, if needed. =item I is a switch to turn ODBC connection on or off =item I to use a different data source name when ODBC is used An example file F should have come with the sources (or the Debian package); please consult this file for more examples. =back =head2 ODBC CONFIGURATION There are now several ODBC systems available for Linux / Unix. The following F<~/.odbc.ini> work with the B library and the B ODBC driver on my Debian GNU/Linux system: [ODBC Data Sources] beancounter = BeanCounter Database [beancounter] Driver = /usr/lib/libpsqlodbc.so Database = beancounter Servername = localhost [ODBC] InstallDir = /usr/lib Alternatively, the B library can be used with the following scheme for F (or F<~/.odbcinst.ini>) to define the Postgres database drivers [PostgreSQL] Description = PostgreSQL ODBC driver for Linux and Windows Driver = /usr/lib/postgresql/lib/libodbcpsql.so Setup = /usr/lib/odbc/libodbcpsqlS.so Debug = 0 CommLog = 0 FileUsage = 1 after which F (or F<~/.odbc.ini>) can be used to define actual data sources as follows: [PostgreSQL] Description = PostgreSQL template1 Driver = PostgreSQL Trace = No TraceFile = /tmp/odbc.log Database = template1 Servername = localhost UserName = Password = Port = 5432 Protocol = 6.4 ReadOnly = Yes RowVersioning = No ShowSystemTables= No ShowOidColumn = No FakeOidIndex = No ConnSettings = [beancounter] Description = Beancounter DB (Postgresql) Driver = Postgresql Trace = No TraceFile = Database = beancounter Servername = some.db.host.com UserName = Password = Port = 5432 Protocol = 6.4 ReadOnly = No RowVersioning = No ShowSystemTables= No ShowOidColumn = No FakeOidIndex = No ConnSettings = =head1 BUGS B and B are so fresh that there are only missing features :) Seriously, check the TODO list. This code or its predecessors have been used by the author since the end of 1998. =head1 SEE ALSO F, F, F, F, F, F, F, F. =head1 COPYRIGHT beancounter is (c) 2000 - 2006 by Dirk Eddelbuettel Updates to this program might appear at F. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. There is NO warranty whatsoever. The information that you obtain with this program may be copyrighted by Yahoo! Inc., and is governed by their usage license. See F for more information. Equivalently, foreign exchange rates from F are for academic research and teaching. See F for more details. =head1 ACKNOWLEDGEMENTS The Finance::YahooQuote module, originally written by Dj Padzensky (and on the web at F as well as at F) serves as the backbone for data retrieval, which was also already very useful for the real-time ticker F. =cut beancounter-0.8.10/example.beancounterrc0000644000000000000000000000444610011323167015204 0ustar # # example resource file ~/.beancounterrc for BeanCounter # # # beancounter --- A stock portfolio performance monitoring tool # # Copyright (C) 1998 - 2004 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # # # The format of this file is simple: pairs of key=value where # the key must be a symbol used and recognised by the global hash # %Config; some command-line arguments are also recognized. See # the GetConfig() function in BeanCounter.pm for the gory details. # # Set the host on which the beancounter db is stored host=localhost # Set the database user -- Note that you if don't specify # a user, your (system) uid is used via the USER variable #user=joe # Set the database password -- not needed if e.g. the database # grants eg access locally. Hence defaults to 'undef'. #passwd=joesecret # Declare that we want to connect using ODBC rather than via the # Perl DBD driver for PostgreSQL or MySQL. See beancounter(1) for an # example ODBC configuration file, and the libiodbc documentation. # Default is 0, i.e. DBD-Pg or DBD-mysql. #odbc=1 # ODBC datasource name, defaults to beancounter #dsn=beancounter # Set the default domestic currency. Default is USD (yes, I moved :) #currency=CAD # Example of setting a firewall proxy host, and user:password #proxy=http://1.2.3.4:80 #firewall=joeuser:joepasswd # Debugging #debug=0 # Verbose operation #verbose=0 # Database system, eg to select MySQL over the default PostgreSQL #dbsystem=PostgreSQL # Database name, eg to select something other than the default beancounter #dbname=beancounter # Shall FX rates be updated? Default is yes #fxupdate=1 beancounter-0.8.10/setup_beancounter0000755000000000000000000004541211405254334014455 0ustar #! /bin/bash -e # # setup_beancounter --- Example usage of the beancounter tool # # Copyright (C) 2000 - 2006 Dirk Eddelbuettel # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program; if not, write to the Free Software # Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA. # $Id: setup_beancounter,v 1.41 2006/03/22 04:15:28 edd Exp $ # If we really want to be independent of the underlying database, this # probably needs to written in Perl using DBI/DBD # # This once started as being PostgreSQL specific ... # # MySQL patches by Mathias Weidner # # SQLite support was added some time in 2003. # # As of 2005-04-13, two SQLite versions are supported: the new SQLite3 is the # default, but with option -o $file, the previous version can be used. This # assumes a layout as on Debian systems, i.e. # # current v3 previous v2 # binary programs sqlite3 sqlite # Perl DBI Module DBD::SQLite DBD::SQLite2 # # i.e. a binary program 'sqlite3' is used to create a database for SQLite 3.* # for which Perl code uses the DBD::SQLite module --- whereas 'sqlite' invokes # the binary for the previous release SQLite 2.* for which Perl uses DBD::SQLite2 # If this differs on your system, you need to adjust the assignments of # DBCOMMAND below, and/or adjust BeanCounter.pm for the proper Perl module. # -------------------------- 'global' variables progname=$(basename $0) if [ "$USER" = "" ] then USER=$(whoami) fi DATABASE='beancounter' VERSION='0.8.10' DBSYSTEM='PostgreSQL' #DBSYSTEM='MySQL' SCHEMA_ONLY=0 DAYOFWEEK=$(date +"%u") if [ "$DAYOFWEEK" -gt 5 ] || [ "$DAYOFWEEK" -eq 1 ] then LASTBIZDAY="last friday" else LASTBIZDAY="yesterday" fi # Dow Jones Industrial Average # see for example http://www.amex.com/indexshares/index_shares_diamonds_list.stm DJIA="AA AXP T BA CAT C KO DIS DD EK XOM GE GM HPQ HD HON INTC IBM IP JNJ MCD MRK MSFT MMM JPM MO PG T UTX WMT" # S&P500 is at http://www.spglobal.com/ssindexmain500text.html # NASDAQ is at http://www.nasdaq.com/asp/nasdaq100_activity.stm # S&P/TSE60 is at http://www.spglobal.com/ssindexmaintsealpha.html # (NB that Canadian stocks need a .TO suffix, eg Air Canada becomes AC.TO) # --------------------------- shell functions below -------------------------- function usage_and_exit { cat</dev/null 2>&1 if [ $? -eq 1 ] then echo "" echo "*** Error: No postgresql user '$1'" echo "" echo "We were unable to start psql as the user '$1' does not exist" echo "You need to create a Postgresql user '$1' first:" echo "" echo " Change to user postgres: $ su - postgres" echo " Create the user: # createuser $1 -d -A" echo " Exit from user postgres: # exit" echo "" echo "and then run this script again." echo "" exit 1 # else # echo "Good: Postgresql can be accessed by $1" fi set -e } function exit_if_no_mysql_user { set +e ( echo "select user from user;" | \ $DBCOMMAND mysql | grep $1 ) >/dev/null 2>&1 if [ $? -eq 1 ] then echo "" echo "*** Error: No mysql user '$1'" echo "" echo "We were unable to start mysql as the user '$1' does not exist" echo "You need to create a MySQL user '$1' first:" echo "" echo " Change to super user (MySQL Admin): $ su -" echo " Start mysql program with mysql db: # mysql mysql" echo " Create the user: grant all privileges on *.* to $USER@localhost;" echo " Exit from user mysql: > \q" echo " Exit from super user: # exit" echo "" echo "and then run this script again." echo "" echo "[ NB These instructions are currently approximative. Consult your manual. ]" echo "[ And email me a better version :) ]" exit 1 # else # echo "Good: MySQL can be accessed by $1" fi set -e } # ------------------------ 'main' code ------------------------------------- while getopts ":mn:l:o:sh" opt do case $opt in m) DBSYSTEM='MySQL' echo "Now using $DBSYSTEM" ;; n) DATABASE=$OPTARG echo "Now using database name $DATABASE" ;; l) DBSYSTEM='SQLite' DATABASE=$OPTARG DBCOMMAND="sqlite3" echo "Now using (current) SQLite3 on $DATABASE" ;; o) DBSYSTEM='SQLite2' DATABASE=$OPTARG DBCOMMAND="sqlite" echo "Now using (old) SQLite on $DATABASE" ;; s) SCHEMA_ONLY=1 ;; h) usage_and_exit ;; ?) echo "Ignoring unknown argument, try '$progname -h' for help." ;; esac done BEANCOUNTER="beancounter --dbsystem $DBSYSTEM --dbname $DATABASE" if [ "$DBSYSTEM" = "PostgreSQL" ]; then # if Postgres if [ -z "$PASSWORD" ]; then DBCOMMAND="psql -q" else DBCOMMAND="psql -q -W $PASSWORD" fi elif [ "$DBSYSTEM" = "MySQL" ]; then # if MySQL # mysql(1) arguments -- you could add host, port, ... here if [ -z "$PASSWORD" ]; then DBCOMMAND="mysql" else DBCOMMAND="mysql -p$PASSWORD" fi elif [ "$DBSYSTEM" = "SQLite" ]; then # if SQLite v3 true # nothing to do elif [ "$DBSYSTEM" = "SQLite2" ]; then # if SQLite v2 true # nothing to do else echo "" echo "*** Error: Unsupported database system ***" echo "" echo "The backend $DBSYSTEM is not supported. Patches welcome..." echo "" exit 1 fi # test if we are running this as root exit_if_root $USER # test if database can be accessed by user if [ "$DBSYSTEM" = "PostgreSQL" ]; then exit_if_no_postgres_user $USER elif [ "$DBSYSTEM" = "MySQL" ]; then exit_if_no_mysql_user $USER fi # test if $DATABASE exists and exit if true exit_if_exists if [ "$DBSYSTEM" = "PostgreSQL" ] || [ "$DBSYSTEM" = "MySQL" ]; then echo "Creating $DATABASE database" if [ "$DBSYSTEM" = "PostgreSQL" ]; then execute "createdb $DATABASE" elif [ "$DBSYSTEM" = "MySQL" ]; then echo "create database $DATABASE;" | $DBCOMMAND # NB deleting db under MySQL is: mysqladmin drop beancounter_test -p$PASSWORD # grant access to $DATABASE db #GRANT="grant select,delete,insert,update on $DATABASE.* to $USER@localhost" #echo $GRANT | $DBCOMMAND fi echo "" fi echo "Creating $DATABASE database tables" if [ "$DBSYSTEM" = "PostgreSQL" ]; then create_postgres_tables elif [ "$DBSYSTEM" = "MySQL" ]; then create_mysql_tables elif [ "$DBSYSTEM" = "SQLite2" ] || [ "$DBSYSTEM" = "SQLite" ]; then create_sqlite_tables fi if [ "$SCHEMA_ONLY" -eq 1 ]; then echo "Schema created, exiting" exit 0 fi if [ "$DBSYSTEM" = "PostgreSQL" ] || [ "$DBSYSTEM" = "MySQL" ]; then # testing database access from Perl echo "Verifying database access from Perl" set +e $BEANCOUNTER checkdbconnection rc="$?" if [ "$rc" != 1 ] then echo "Failure --- please check database permission, possible" echo "requirement of a password and other sources of failure" echo "to establish a connection." exit 1 fi set -e fi # insert the DJIA stocks into the indices table echo "Filling $DATABASE database tables with DJIA stocks" execute "$BEANCOUNTER addindex DJIA $DJIA" # create an example of tech stocks echo "Filling $DATABASE (sample) portfolio" #execute "$BEANCOUNTER addportfolio CSCO:50:USD NT.TO:100:CAD SIE.DE:10:EUR CGE.PA:50:EUR" execute "$BEANCOUNTER addportfolio IBM:50:USD XOM:75:USD C:100:USD GOOG:25:USD" echo -n "Filling $DATABASE with stock info and most recent prices " echo "for DJIA stocks" execute "$BEANCOUNTER addstock $DJIA ^GSPC" echo "Filling $DATABASE with historical prices for example portfolio stocks" #execute "$BEANCOUNTER backpopulate --prevdate '1 year ago' --date '$LASTBIZDAY' CSCO NT.TO SIE.DE CGE.PA" execute "$BEANCOUNTER backpopulate --prevdate '1 year ago' --date '$LASTBIZDAY' IBM XOM C GOOG ^GSPC" echo "Filling $DATABASE with historical fx prices for EUR and CAD" execute "$BEANCOUNTER fxbackpopulate --ubcfx --prevdate '1 year ago' --date '$LASTBIZDAY' EUR CAD" echo "Running portfolio pl report on (sample) portfolio" execute "$BEANCOUNTER plreport --date '$LASTBIZDAY'" # dayendreport needs a db entry previous price which we don't get from # all backpop routines / data combinations #echo "Running portfolio dayendreport on (sample) portfolio" #execute "$BEANCOUNTER dayendreport --date '$LASTBIZDAY'" echo "Running portfolio riskreport on (sample) portfolio" execute "$BEANCOUNTER risk --date '$LASTBIZDAY'" echo "Done." exit 0 =head1 NAME setup_beancounter - Create and initialise BeanCounter databases =head1 SYNOPSIS setup_beancounter [-m] [-n NAME] [-n] [-h] =head1 DESCRIPTION B creates and initialises the databases used by B. It also runs some initial reports to illustrate some of the features of B. =head1 OPTIONS -m Use MySQL as the backend over the default PostgreSQL -l dbfile Use SQLite \(version 3 or later\) with database file 'dbfile' -o dbfile Use SQLite2 \(compatibility mode\) with database file 'dbfile' -n name Use name as the database instead of beancounter -s Do not fill the database, only create its schema -h Show a simple help message =head1 SEE ALSO B(1), B(1), B(1) =head1 AUTHOR Dirk Eddelbuettel edd@debian.org =cut beancounter-0.8.10/t/0000755000000000000000000000000010021246003011222 5ustar beancounter-0.8.10/t/01base.t0000755000000000000000000000024110021245676012501 0ustar #!/usr/bin/perl print "1..$tests\n"; require Finance::BeanCounter; print "ok 1\n"; import Finance::BeanCounter; print "ok 2\n"; BEGIN{$tests = 2;} exit(0); beancounter-0.8.10/beancounter.html0000644000000000000000000007725010410150173014171 0ustar beancounter - Stock portfolio performance monitor tool

NAME

beancounter - Stock portfolio performance monitor tool

SYNOPSYS

beancounter [options] command [command_arguments ...]

COMMANDS

 addindex index args       add stock(s) to market index 'indx'
 addportfolio sym:nb:fx:type:o:pp:pd ... 
                           add 'nb' stocks of company with symbol 'sym'
                           that are listed in currency 'fx' to the 
                           portfolio with optional 'type' and 'owner'
                           info, purchase price 'pp' and date 'pd'; 
                           see below for a complete example
 allreports                combines dayendreport, status and risk 
 addstock arg ...          add stock(s) with symbol arg to the database
 advancement               report on unrealized gains from lows
 backpopulate  arg ...     fill with historic data for given stock(s)
 checkdbconnection         test if connection to db can be established
 dailyjob                  combines update, dayendreport, status + risk 
 dayendreport              reports p/l changes relative to previous day
 deactivate symbol ...     set stock(s) inactive in stockinfo table
 delete arg ...            delete given stock(s) from database
 destroydb                 delete the BeanCounter database
 fxbackpopulate  arg ...   fill with historic data for currency(ies)
 lsportfolio               list portfolio data
 plreport                  run an portfolio p/l report rel. to any day
 quote arg ...             report current data for given stock(s)
 retracement               report unrealized losses from highs (drawdowns)
 risk                      display a portfolio risk report
 split arg ...             split-adjust price history and portfolio
 status                    status summary report for portfolio
 update                    update the database with day's data
 warranty                  display the short GNU GPL statement

OPTIONS

 --help                    show this help
 --verbose                 more verbose operation, debugging
 --date date               report for this date (today)
 --prevdate date           relative to this date (yesterday)
 --currency fx             set home currency
 --restriction sql         impose SQL restriction
 --extrafx fx1,fx2,...     additional currencies to load
 --forceupdate date        force db to store new price info with date
 --rcfile file             use different configuration file
 --[no]fxupdate            enforce/suppress FX update, default is update
 --[no]commit              enforce/suppress database update, default is commit
 --[no]equityupdate        enforce/suppress Equity update, default is update
 --[no]ubcfx               use/skip FX from UBC's Sauder school, default skip
 --splitby arg             split stock history + position by this factor [2]
 --dbsystem system         use db backend system, default is PostgreSQL
 --dbname name             use db name, default is beancounter

DESCRIPTION

beancounter gathers and analyses stock market data to evaluate portfolio performance. It has several modes of operation. The first main mode is data gathering: both current data (e.g. end-of-day closing prices) and historical price data (to back-populate the database) can be retrieved both automatically and efficiently with subsequent local storage in a relational database system (either PostgreSQL, MySQL or SQLite) though any other system with an ODBC driver could be used). The second main mode is data analysis where the stored data is evaluated to provide performance information. Several canned reports types are already available.

Data is retrieved very efficiently in a single batch query per Yahoo! host from the Yahoo! Finance web sites using Finance::YahooQuote module (where version 0.18 or newer is required for proxy support). Support exists for North America (i.e. US and Canada), Europe (i.e. the Continent as well as Great Britain), several Asian stock markets, Australia and New Zealand.

beancounter can aggregate the change in value for the entire portfolio over arbitrary time horizons (provided historical data has either been gathered or has been backpopulated). Using the powerful date-parsing routine available to Perl (thanks to the Date::Manip modules), you can simply say 'from six months ago to today' (see below for examples).

beancounter has been written and tested under Linux. It should run under any standard Unix as long as the required Perl modules are installed, as as long as the database backend is found.

EXAMPLES

 beancounter update --forceupdate today
    This updates the database: it extends timeseries data (such as
    open, low, high, close, volume) with data for the current day,
    and overwrites static data (such as capital, price/earnings, ...) 
    with current data. All stocks held in the database are updated
    (unless the --restriction argument instructs otherwise). The 
    --forceupdate option lets the program corrects incorrect dates 
    returned from Yahoo! (which happens every now and so often), but
    be careful to correct for this on public holidays. Note that 
    the --restriction argument will be applied to the portfolio table,
    whereas the overall selection comes from the stockinfo table.
 beancounter addportfolio SUNW:100:USD:401k:joe:85.50:19991117 \
                          IBM:100:USD:401k:joe:90.25:20000320  \
                          SPY:50:USD:ira:joe:142.25:20000620
    This adds the two stocks Sun and IBM to the 401k portfolio of Joe,
    as well as SP500 'Spiders' to his IRA portfolio. The stocks are
    also added to the general stock info tables via an implicit call
    of the stockinfo command.
 beancounter addstock LNUX RHAT COR.TO
    This adds these three Linux companies to the database without adding
    them to any specific portfolios.
 beancounter backpopulate --prevdate '1 year ago' \
                          --date 'friday 1 week ago' IBM SUNW HWP
    This backpopulates the database with historic prices for three
    hardware companies. Note how the date specification is very general
    thanks to the underlying Date::Manip module.
 beancounter fxbackpopulate --prevdate '1 year ago' \
                          --date 'friday 1 week ago' CAD EUR GBP
    This backpopulates the database with historic prices for these
    three currencies. Note how the date specification is very general
    thanks to the underlying Date::Manip module.
    Unfortunately, Yahoo! is a little bone-headed in its implementation
    of historic FX rates -- these are stored to only two decimals 
    precision, just like stockprices. Unfortunately, convention is to
    use at least four if not six. Because of the limited information, 
    risk from FX changes will be underestimated.
 beancounter plreport --prevdate '1 month ago' --date 'today' \
                        --restriction "owner='joe'"
    This calculates portfolio profits or losses over the last month. It
    also imposes the database restriction that only stocks owned by
    'joe' are to be included.
 beancounter status --restriction "type='401k'"
    This shows a portfolio status report with the restriction that only
    stocks from the '401k' account are to be included.
 beancounter risk --prevdate "6 month ago"
    This shows a portfolio risk report. This tries describes the 
    statistically plausible loss which should be exceeded only 1 out
    of 100 times (see below for more details).
 beancounter dailyjob --forceupdate today
    Run a complete 'job': update the database, show a day-end profit/loss
    report, show a portfolio status report and show a riskreport. In the
    update mode, override a potentially wrong date supplied by Yahoo!
    with the current date.
 beancounter split --splitby 3 --prevdate 1990-01-01 ABC CDE
    Split-adjusts the (hypothetical) stocks ABC and CDE by a factor
    of three: price data in the database is divided by three, volume
    increased by 3 and similarly, in the portfolio shares are increased
    and cost is descreased.  Default dates are --prevdate and --date
    which may need adjusting.

TUTORIAL

The following few paragraphs will illustrate the use of beancounter. We will set up two fictional accounts for two brothers Bob and Bill (so that we can illustrate the 'owner' column). The prices below are completely fictitious, as are the portfolios.

We suppose that beancounter is installed and that the setup_beancounter command has been run. We can then create a two-stock (computer hardware) portfolio for Bob as follows:

 beancounter addportfolio SUNW:100:USD:401k:bob:85.50:19991117 \
                          IBM:100:USD:401k:bob:90.25:20000320

Here we specify that 100 shares each of Sun and IBM, priced in US Dollars, are in Bob's portfolio which is tagged as a 401k retirement account. The (fictitious) purchase price and date are also given.

Let's suppose that Bill prefers networking equipment, and that he has a brokerage account in Canada:

 beancounter addportfolio CSCO:100:USD:spec:bill:78.00:19990817 \
                          NT:200:CAD:spec:bill:cad:90.25:20000212

Now we can backpopulate the database from 1998 onwards for all four stocks:

 beancounter backpopulate --prevdate 19980101 CSCO IBM NT SUNW

With this historical data in place, we now compare how Bob's portfolio would have fared over the last 18 months:

 beancounter plreport --prevdate '18 months ago' \
                        --restriction "owner='bob'"

Note how we use double quotes to protect the arguments, and how the SQL restriction contains a further single quote around the literal string.

We can also review the performance for Bill at the most recent trading day:

 beancounter dayendreport --restriction "owner='bill'"

or the status of holdings and their respective values:

 beancounter dayendreport --restriction "owner='bill'"

Similarly, a risk reports can be run on this portfolios per

 beancounter risk --restriction "owner='bill'"

MORE DETAILED COMMAND DESCRIPTION

addportfolio is the most important 'position entry' command. As with other commands, several arguments can be given at the same time. For each of these, records are separated using a colon and specify, in order, stock symbol, number of stocks held, currency, account type, account owner, purchase price and purchase date. Only the first three arguments are required, the others are optional. Executing addportfolio implicitly executes addstock. The account type column can be used to specify whether the account is, e.g., a tax-sheltered retirement account, or it could be used to denote the brokerage company is it held at.

plreport retrieves the most recent quotes(s). This is useful for illiquid securities which might not have traded that day, or if a public holiday occurred, or if there was a data error at Yahoo!. Two dates can be specified which determine the period over which the profit or loss is computed. This will fail if price data (or currency data in the case of foreign stocks data) data is not available for either of those two dates. This may be restrictive for foreign stocks where we cannot backpopulate due to lack of public data source for historical currency quotes. Major currencies can be retrieved from Yahoo!, but only to two decimals precisions.

dayendreport is similar to plreport but is always over a one-day period. It also uses only one date record by calculating performance given the 'previous close' data.

status shows holdings amounts, total position values, annualized returns in percentages and holding periods in days. Note that the annualized returns can appear excessive if, e.g., a ten-day return from a recently purchased stock is extrapolated to an annual time period.

risk shows a portfolio risk report which describes the statistically plausible loss which should be exceeded only 1 out of 100 times. In other words, the loss estimate has a critical level of 99%. This risk level is estimated via two methods. The first is non-parametric and assumes no particular model or distribution; it computes the 1% quintile of the return distribution and displays it as well as the corresponding asset value at risk. The second method uses the standard Value-at-Risk (VaR) approach. This uses the 1% critical value of the Normal distribution and implicitly assumes a normal distribution for returns. See http://www.gloriamundi.org for more introduction and references. If the distribution of normalitty was perfectly true, both measures would coincide. A large difference between the two estimates would indicate that the return distribution might be rather non-normal. Another view of the riskiness of a given position is provided by the last column with the 'margVaR' heading. It shows the marginal Value-at-Risk. Marginal VaR is commonly defined as the risk contribution of the given position to the total portfolio, and calculated as the difference in the VaR of the full portfolio and the VaR of an otherwise identical portfolio with the given position removed. Note that calculating marginal VaR is fairly slow (on the order of O(n^3) ].

retracement shows a 'drawdown' report. Drawdown is commonly defined as the percentage loss relative to the previous high. The default period is used, but can be altered with the --date and --prevdate options. The default period is also corrected for the actual holding period. In other words, if a stock has been held for two months, only those two months are used instead of the default of six months -- but if the last months has been selected via --prevdate then it is used. For short positions, the analysis is inverted and relative to the previous low. The report displays each stock, the number of shares held, the current price and holdings value. The next two columns show the maximum price attained in the examined period, and the percent decline relative to it. The last column shows the unrealized loss relative to the maximum price over the period. The aggregate holdings value, percent decline and unrealized loss are shown as well.

advancement does the opposite of drawdown -- it computes unrealized gains relative to the minimum price in the period. The discussion in the preceding paragraph applies `but inverted'.

lsportfolio simply lists the content of the portfolio table. A SQL restriction can be imposed.

addindex adds stocks a the index table. Currently, no further analysis references this table.

addstock adds stocks to the database. From then on data will be retrieved for the given symbol(s) and stored in the database whenever the update command is executed.

backpopulate fills the database with historic prices for the given symbols and date period. Note that this works well for stocks and mutual fund. Options have no historic data stored. Currencies are stored with limited precision as noted above.

quote simply shows a price quote for the given symbol(s).

update updates the database with quotes for all stocks for the given day. No output is generated making the command suitable for cron execution.

dailyjob is a simple convenience wrapper around update, dayendreport, status and risk,

allreports is a another covenience wrapper around dayendreport, status and risk.

deactivate will set the active column in stockinfo for the given symbol(s) to false thereby inhibiting any further updates of symbol(s). The existing data for symbol(s) is retained. Use this when a stock is acquired, delisted, or you simply want to stop tracking it -- but do not want to purge the historical data.

split adjusts the price database, and the portfolio holdings, for stock splits. The default factor is 2, this can be adjusted with the option --splitby. The dates arguments can be set with --prevdate and --date.

delete removes the given symbols from the database.

destroydb deletes the BeanCounter database.

checkdbconnection simply opens and closes the database handle, and returns a specified exit code which can then be tested. This is used in the setup_beancounter command.

warranty display a short GNU General Public License statement.

MORE DETAILED OPTION DESCRIPTION

--currency can be used to select a different home currency. Instead of having all values converted to the default currency, the selected currency is used.

--date allows to choose a different reference date. This is then be be used by commands working on a date, or date period, such as plreport, dayendreport, backpopulate, fxbackpopulate or status. --prevdate allows to choose a different start date for return calculations, or data gathering.

--restriction can be used to restrict the database selection. The argument must be a valid part of valid SQL statement in the sense that existing columns and operators have to be employed. The argument to this option will be completed with a leading and. The SQL restriction will typcally be over elements of the portfolio table which comprises the columns symbol, shares, currency, type, owner, cost and date. A simple example would be currency='CAD'. Note that this has to protected by double quotes ``I on the command-line.

--extrafx allows to gather data on additional currency rates beyond those automatically selected as shares are listed in them. A typical example would be for a European investor wanting to convert from the EUR in which the shares are listed into one of the member currencies which beancounter would no longer retrieve as shares are no longer listed in these.

--forceupdate allows to overwrite an potentially wrong date in the database update. Unfortunately, it appears that Yahoo! occasionally reports correct prices with an incorrect date such as the previous day's. In such a case, this option, along with an argument such as 'today' can override the bad date datapoint and avoid a hole in the database. The downside of this approach is that it would ``double'' the previous data in the case of a public holiday, or even if it was run the weekend. A somewhat smarter comparison to previously stored data might prevent that, but would be more complex to implement.

--rcfile allows to specify a resource file different from the default ~/.beancounterrc.

--dbsystem allows to switch to a different database backend. The default is PostgreSQL but MySQL and SQLite are also supported. For SQLite, the default is now version 3.* but the previous version -- which is not binarily compatible -- is supported as well with argument 'SQLite2'.

--dbsystem allows to switch to an alternate database. The default is 'beancounter'. This can be useful for testing new features.

--fxupdate is a boolean switch to enforece updates of FX rates during 'update'. The default is 'true' but '--nofxupdate' can be used to suppress the update of foreign exchange rates.

Similarly, --equityupdate is a boolean switch to enforece, or suppress updates of Equity (i.e. stock) data during 'update'. The default is 'true' but '--noequityupdate' can be used to suppress the update of foreign exchange rates.

--ubcfx is a boolean switch to use the 'PACIFIC' FX rate service from the Sauder School at UBC. This is useful when the default FX rate service at Yahoo! is erratic, or unreliable. While the PACIFIC server provides a wider variety of exchange rates, Yahoo! can still be useful as it can provide more columns (open/high/low). However, during most of 2005, Yahoo! has been unrealiable for the exchange rates and has not provided historical FX data. On the other hand, the UBC service does not run on Canadian holidays so it cannot really server as a full substitute. Contributions for a new data acquisition, maybe via www.oanda.com would be welcome.

--splitby can be used to set a stock split factor other than the default of 2.

--host can be used to point to a machine containing the PostgreSQL or MySQL database. The machine can be remote, or it can be the actual machine beancounter is running on. If a hostname is given, tcp/ip connection are used. If no hostname is given, the default value of 'localhost' implies that local socket connections are used which may be easier to employ for less experienced adatabase users.

Also, --commit is a boolean switch to suppress actual database updates if the negated --nocommit is selected. This is useful mostly in debugging contexts.

The --verbose and --debug switches can be used in debugging an testing, and --help triggers the display of help message.

SYSTEM OVERVIEW

The following section details some of the database and configuration options.

DATABASE REQUIREMENTS

beancounter currently depends on either PostgreSQL, MySQL, SQLite (version 2 or 3) or any other database for which an ODBC driver is available (though the required tables would have to created manually in the ODBC case). Yet another DB backend could be added provided suitable Perl DBI drivers are available. For PostgreSQL, MySQL and SQLite, the setup_beancounter script can create and initialize the database, form the required tables and fills them with some example data. It is a starting point for local modifications.

The connection to the database is made via a dedicated function in the BeanCounter.pm module, changes would only have to be made there. As of this writing the Perl DBI (the database-independent interface for Perl) is used along the DBI drivers for PostgreSQL, MySQL, SQLite and ODBC. Ports for Oracle, Sybase, ... are encouraged.

CONFIG FILE

A configuration file ~/.beancounterrc is read if found. It currently supports the following options:

currency to specify into which home currency holdings and profits/losses have to be converted
host to specify the database server on which the BeanCounter database resides (this is needed only for the alternate connection via the DBI-Pg driver in case DBI-ODBC is not used)
user to specify the userid for the database connection; if needed. If not specified, the current user id is used.
passwd to specify the password for the database connection, if needed.
dbsystem to select a database backend, e.g. to switch from PostgreSQL to MySQL or SQLite or SQLite2 (the previous format of SQLite).
dbsystem to select a different default database name other than the default of 'beancounter'
proxy to specify the address of a firewall proxy server if one is needed to connect to the Internet.
firewall to specify a firewallid:firewallpasswd combination, if needed.
odbc is a switch to turn ODBC connection on or off
dsn to use a different data source name when ODBC is used

An example file example.beancounterrc should have come with the sources (or the Debian package); please consult this file for more examples.

ODBC CONFIGURATION

There are now several ODBC systems available for Linux / Unix. The following ~/.odbc.ini work with the iODBC library and the PostgreSQL ODBC driver on my Debian GNU/Linux system:

   [ODBC Data Sources]
   beancounter = BeanCounter Database
   [beancounter]
   Driver       = /usr/lib/libpsqlodbc.so
   Database     = beancounter
   Servername   = localhost
   [ODBC]
   InstallDir = /usr/lib

Alternatively, the unixODBC library can be used with the following scheme for /etc/odbcinst.ini (or ~/.odbcinst.ini) to define the Postgres database drivers

   [PostgreSQL]
   Description     = PostgreSQL ODBC driver for Linux and Windows
   Driver          = /usr/lib/postgresql/lib/libodbcpsql.so
   Setup           = /usr/lib/odbc/libodbcpsqlS.so
   Debug           = 0
   CommLog         = 0
   FileUsage       = 1

after which /etc/odbc.ini (or ~/.odbc.ini) can be used to define actual data sources as follows:

   [PostgreSQL]
   Description     = PostgreSQL template1
   Driver          = PostgreSQL
   Trace           = No
   TraceFile       = /tmp/odbc.log
   Database        = template1
   Servername      = localhost
   UserName        =
   Password        =
   Port            = 5432
   Protocol        = 6.4
   ReadOnly        = Yes
   RowVersioning   = No
   ShowSystemTables= No
   ShowOidColumn   = No
   FakeOidIndex    = No
   ConnSettings    =
   [beancounter]
   Description     = Beancounter DB (Postgresql)
   Driver          = Postgresql
   Trace           = No
   TraceFile       =
   Database        = beancounter
   Servername      = some.db.host.com
   UserName        =
   Password        =
   Port            = 5432
   Protocol        = 6.4
   ReadOnly        = No
   RowVersioning   = No
   ShowSystemTables= No
   ShowOidColumn   = No
   FakeOidIndex    = No
   ConnSettings    =

BUGS

Finance::BeanCounter and beancounter are so fresh that there are only missing features :) Seriously, check the TODO list. This code or its predecessors have been used by the author since the end of 1998.

SEE ALSO

Finance::BeanCounter.3pm, smtm.1, Finance::YahooQuote.3pm, LWP.3pm, Date::Manip.3pm, Statistics::Descriptive.3pm, setup_beancounter.1, update_beancounter.1.

COPYRIGHT

beancounter is (c) 2000 - 2006 by Dirk Eddelbuettel <edd@debian.org>

Updates to this program might appear at http://dirk.eddelbuettel.com/code/beancounter.html.

This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. There is NO warranty whatsoever.

The information that you obtain with this program may be copyrighted by Yahoo! Inc., and is governed by their usage license. See http://www.yahoo.com/docs/info/gen_disclaimer.html for more information.

Equivalently, foreign exchange rates from http://fx.sauder.ubc.ca are for academic research and teaching. See http://fx.sauder.ubc.ca/about.html for more details.

ACKNOWLEDGEMENTS

The Finance::YahooQuote module, originally written by Dj Padzensky (and on the web at http://www.padz.net/~djpadz/YahooQuote/ as well as at http://dirk.eddelbuettel.com/code/yahooquote) serves as the backbone for data retrieval, which was also already very useful for the real-time ticker http://dirk.eddelbuettel.com/code/smtm.html.