dballe-8.6/0000755000175000017500000000000013602152021007603 500000000000000dballe-8.6/missing0000755000175000017500000001533613602151750011142 00000000000000#! /bin/sh # Common wrapper for a few potentially missing GNU programs. scriptversion=2018-03-07.03; # UTC # Copyright (C) 1996-2018 Free Software Foundation, Inc. # Originally written by Fran,cois Pinard , 1996. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. if test $# -eq 0; then echo 1>&2 "Try '$0 --help' for more information" exit 1 fi case $1 in --is-lightweight) # Used by our autoconf macros to check whether the available missing # script is modern enough. exit 0 ;; --run) # Back-compat with the calling convention used by older automake. shift ;; -h|--h|--he|--hel|--help) echo "\ $0 [OPTION]... PROGRAM [ARGUMENT]... Run 'PROGRAM [ARGUMENT]...', returning a proper advice when this fails due to PROGRAM being missing or too old. Options: -h, --help display this help and exit -v, --version output version information and exit Supported PROGRAM values: aclocal autoconf autoheader autom4te automake makeinfo bison yacc flex lex help2man Version suffixes to PROGRAM as well as the prefixes 'gnu-', 'gnu', and 'g' are ignored when checking the name. Send bug reports to ." exit $? ;; -v|--v|--ve|--ver|--vers|--versi|--versio|--version) echo "missing $scriptversion (GNU Automake)" exit $? ;; -*) echo 1>&2 "$0: unknown '$1' option" echo 1>&2 "Try '$0 --help' for more information" exit 1 ;; esac # Run the given program, remember its exit status. "$@"; st=$? # If it succeeded, we are done. test $st -eq 0 && exit 0 # Also exit now if we it failed (or wasn't found), and '--version' was # passed; such an option is passed most likely to detect whether the # program is present and works. case $2 in --version|--help) exit $st;; esac # Exit code 63 means version mismatch. This often happens when the user # tries to use an ancient version of a tool on a file that requires a # minimum version. if test $st -eq 63; then msg="probably too old" elif test $st -eq 127; then # Program was missing. msg="missing on your system" else # Program was found and executed, but failed. Give up. exit $st fi perl_URL=https://www.perl.org/ flex_URL=https://github.com/westes/flex gnu_software_URL=https://www.gnu.org/software program_details () { case $1 in aclocal|automake) echo "The '$1' program is part of the GNU Automake package:" echo "<$gnu_software_URL/automake>" echo "It also requires GNU Autoconf, GNU m4 and Perl in order to run:" echo "<$gnu_software_URL/autoconf>" echo "<$gnu_software_URL/m4/>" echo "<$perl_URL>" ;; autoconf|autom4te|autoheader) echo "The '$1' program is part of the GNU Autoconf package:" echo "<$gnu_software_URL/autoconf/>" echo "It also requires GNU m4 and Perl in order to run:" echo "<$gnu_software_URL/m4/>" echo "<$perl_URL>" ;; esac } give_advice () { # Normalize program name to check for. normalized_program=`echo "$1" | sed ' s/^gnu-//; t s/^gnu//; t s/^g//; t'` printf '%s\n' "'$1' is $msg." configure_deps="'configure.ac' or m4 files included by 'configure.ac'" case $normalized_program in autoconf*) echo "You should only need it if you modified 'configure.ac'," echo "or m4 files included by it." program_details 'autoconf' ;; autoheader*) echo "You should only need it if you modified 'acconfig.h' or" echo "$configure_deps." program_details 'autoheader' ;; automake*) echo "You should only need it if you modified 'Makefile.am' or" echo "$configure_deps." program_details 'automake' ;; aclocal*) echo "You should only need it if you modified 'acinclude.m4' or" echo "$configure_deps." program_details 'aclocal' ;; autom4te*) echo "You might have modified some maintainer files that require" echo "the 'autom4te' program to be rebuilt." program_details 'autom4te' ;; bison*|yacc*) echo "You should only need it if you modified a '.y' file." echo "You may want to install the GNU Bison package:" echo "<$gnu_software_URL/bison/>" ;; lex*|flex*) echo "You should only need it if you modified a '.l' file." echo "You may want to install the Fast Lexical Analyzer package:" echo "<$flex_URL>" ;; help2man*) echo "You should only need it if you modified a dependency" \ "of a man page." echo "You may want to install the GNU Help2man package:" echo "<$gnu_software_URL/help2man/>" ;; makeinfo*) echo "You should only need it if you modified a '.texi' file, or" echo "any other file indirectly affecting the aspect of the manual." echo "You might want to install the Texinfo package:" echo "<$gnu_software_URL/texinfo/>" echo "The spurious makeinfo call might also be the consequence of" echo "using a buggy 'make' (AIX, DU, IRIX), in which case you might" echo "want to install GNU make:" echo "<$gnu_software_URL/make/>" ;; *) echo "You might have modified some files without having the proper" echo "tools for further handling them. Check the 'README' file, it" echo "often tells you about the needed prerequisites for installing" echo "this package. You may also peek at any GNU archive site, in" echo "case some other package contains this missing '$1' program." ;; esac } give_advice "$1" | sed -e '1s/^/WARNING: /' \ -e '2,$s/^/ /' >&2 # Propagate the correct exit status (expected to be 127 for a program # not found, 63 for a program that failed due to version mismatch). exit $st # Local variables: # eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: dballe-8.6/NEWS.md0000644000175000017500000001405113574460113010636 00000000000000# New in version 8.7 * Fixed the command line documentation of possible input types (#202) # New in version 8.6 * Improved python API documentation (#187, #189, #191) * Improved documentation of dballe types (#199) * Turned a segfault into a proper exception (#197) * Parse again '-' as missing (#200) # New in version 8.5 * Fixed passing strings as datetime values from python (#174) # New in version 8.4 * Redesigned and unified all the documentation * B table updates * Allow querying from python without specifying all the datetime min/max values * Fixed querying longitude ranges from python * Fixed reading unset query results from python # New in version 8.3 * Python API: allow other C python extensions to access dballe python objects * B table updates # New in version 8.2 * Python API: fixed `Message.query_station_data` method (#160) * Add level 161 Depth below water surface (#161) # New in version 8.1 * Python API: bindings for core::Data (#158) * Python API: add doc for keywords in cursor classes (#155) * Python API: enabled cursor-based iteration (#154) * Implemented `dbadb --varlist` (#149) # New in version 8.0 * Added `NEWS.md` file * Rationalised keyword usage in Fortran API. See [Input/Output query parameters documentation](doc/fapi_parms.md) for the updated reference. * Added Message, Importer, Exporter, File, DB, Transaction, Cursor\*, Data, Values to the C++ stable API * Dropped support for Python 2 * Dropped support for unused AOF format * Renamed fortran API functions with self-explanatory English names. The old names are kept as aliases for compatibility with existing/old Fortran code * Python API changes: * Added dballe.Message * Added dballe.Importer * Added dballe.Exporter * Added dballe.File * Added dballe.Explorer * Removed dballe.Record in favour of using dict() for input, and using `Cursor*.__getitem__` for output. * Renamed `Cursor*.attr_query` to `Cursor*.query_attrs` * Added `Cursor*.insert_attrs` * Added `Cursor*.remove_attrs` * Added `Cursor*.remove()` * Added `DB.query_messages` * DB or Transaction: `.query_*`: query argument is now optional, omit it to query everything * Added `Cursor.query` and `Cursor.data` * Allow to pass a `Cursor` instead of a `dict` to `insert_data` and `insert_station_data`, to efficiently copy data between databases. * Deprecated `DB.insert_station_data`, `DB.insert_data`, `DB.remove_station_data`, `DB.remove_data`, `DB.query_stations`, `DB.query_station_data`, `DB.query_data`, `DB.query_summary`, `DB.query_messages`, `DB.attr_query_station`, `DB.attr_query_data`, `DB.attr_insert_station`, `DB.attr_insert_data`, `DB.attr_remove_station`, `DB.attr_remove_data`. Use the same methods in `dballe.Transaction` instead. * The use of `?wipe` is now [documented](doc/fapi_connect.mdx) and supported in C++, Python, Fortran, and command line. The value of `wipe` is now parsed to prevent unexpectedly clearing a database when using something like `?wipe=no`. # New in version 7.33 * dballe V7 is now the default (#97) and V6 databases are now unsupported * fixed various bugs (#82, #85, #86, #87, #91, #93, #98, #103, #105, #107) * added new variables to dballe.txt * various performance optimizations (#112, #116) * DBA_FORTRAN_TRANSACTION is now the default (#67) * improved handling of rejected messages and consistency errors * fixed PostgreSQL support when pkg-config is missing * dependency on newer sqlite is now conditional # New in version 7.26 * dballe V7 is now the default (#97) * closed #82, #85, #87, #91, #93, #98 * minor fixes for copr and travis automation * added new variables to dballe.txt # New in version 4.0.4 * Store rep_cod information in generic messages * Allow to create messages without having to set leveltype2 and l2 if not needed * Implemented working with messages with the Fortran API * Refactored the code for the Fortran API * Fixed TOT_PREC12 time range values * Implemented bufrex_msg_reset_sections * Implemented Bufrex::resetSections and same in Python * Test against NaN on setr and setd * Use clean program name in manpages * Store height and pressure as data and not as ana, for flights # New in version 4.0.2 * Fix a compiler error with newer GCC # New in version 4.0.1 * Fix repinfo update * Ported rconvert to new rpy module * Show the currently selected station details in provami, even if it has no data * Decode BUFR edition 2 * Added tables to decode BUFR edition 2 radar reports * Fixed encoding of BUFR flags * Fixed the problem in provami that fails when the only restriction used is the ana filter * Drafted support for pollution data * Use standard floating point huge values to exchange missing values with Fortran * Fortran API: idba_elencamele now properly cleans old values from the output record * Fixed handling of varlist when querying attributes from fortran * Documentation improvements for fapi.tex * Hack a way to sync the values of missing values between Fortran and C * Fixed bufr decoder to work on ARM * Clean records before reading in new data in fortran idba_dammelo # New in version 3.5 * Fixed transactions in cursor.c: now queries don't all happen in a single, huge transaction # Changes from version 2.5 to version 2.6 * dbadb: dbadb export failed to export various kinds of station data * dbadb: implemented dbadb cleanup * dbadb: implemented dbadb stations * dbadb manpage: refer to report code and report memo instead of just 'type' * dbadb manpage: mention what are the query parameters that can be used * dba_var_set* and idba_set* operations now check for overflows and report DBA_ERR_TOOLONG if it happens * fortran API: idba_spiegab now formats the values with the right amount of decimal digits * fortran API: added another predefined callback (idba_error_handle_tolerating_overflows) that only prints a warning in case of overflow * fapi.tex: documented that setting a variable to a missing value is equivalent to an idba_unset * fapi.tex: documented the use of the dballef.h Fortran 90 interface file * README: rewritten to only provide quick start information dballe-8.6/install-sh0000755000175000017500000003601013602151750011537 00000000000000#!/bin/sh # install - install a program, script, or datafile scriptversion=2018-03-11.20; # UTC # This originates from X11R5 (mit/util/scripts/install.sh), which was # later released in X11R6 (xc/config/util/install.sh) with the # following copyright and license. # # Copyright (C) 1994 X Consortium # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to # deal in the Software without restriction, including without limitation the # rights to use, copy, modify, merge, publish, distribute, sublicense, and/or # sell copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # X CONSORTIUM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN # AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNEC- # TION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # # Except as contained in this notice, the name of the X Consortium shall not # be used in advertising or otherwise to promote the sale, use or other deal- # ings in this Software without prior written authorization from the X Consor- # tium. # # # FSF changes to this file are in the public domain. # # Calling this script install-sh is preferred over install.sh, to prevent # 'make' implicit rules from creating a file called install from it # when there is no Makefile. # # This script is compatible with the BSD install script, but was written # from scratch. tab=' ' nl=' ' IFS=" $tab$nl" # Set DOITPROG to "echo" to test this script. doit=${DOITPROG-} doit_exec=${doit:-exec} # Put in absolute file names if you don't have them in your path; # or use environment vars. chgrpprog=${CHGRPPROG-chgrp} chmodprog=${CHMODPROG-chmod} chownprog=${CHOWNPROG-chown} cmpprog=${CMPPROG-cmp} cpprog=${CPPROG-cp} mkdirprog=${MKDIRPROG-mkdir} mvprog=${MVPROG-mv} rmprog=${RMPROG-rm} stripprog=${STRIPPROG-strip} posix_mkdir= # Desired mode of installed file. mode=0755 chgrpcmd= chmodcmd=$chmodprog chowncmd= mvcmd=$mvprog rmcmd="$rmprog -f" stripcmd= src= dst= dir_arg= dst_arg= copy_on_change=false is_target_a_directory=possibly usage="\ Usage: $0 [OPTION]... [-T] SRCFILE DSTFILE or: $0 [OPTION]... SRCFILES... DIRECTORY or: $0 [OPTION]... -t DIRECTORY SRCFILES... or: $0 [OPTION]... -d DIRECTORIES... In the 1st form, copy SRCFILE to DSTFILE. In the 2nd and 3rd, copy all SRCFILES to DIRECTORY. In the 4th, create DIRECTORIES. Options: --help display this help and exit. --version display version info and exit. -c (ignored) -C install only if different (preserve the last data modification time) -d create directories instead of installing files. -g GROUP $chgrpprog installed files to GROUP. -m MODE $chmodprog installed files to MODE. -o USER $chownprog installed files to USER. -s $stripprog installed files. -t DIRECTORY install into DIRECTORY. -T report an error if DSTFILE is a directory. Environment variables override the default commands: CHGRPPROG CHMODPROG CHOWNPROG CMPPROG CPPROG MKDIRPROG MVPROG RMPROG STRIPPROG " while test $# -ne 0; do case $1 in -c) ;; -C) copy_on_change=true;; -d) dir_arg=true;; -g) chgrpcmd="$chgrpprog $2" shift;; --help) echo "$usage"; exit $?;; -m) mode=$2 case $mode in *' '* | *"$tab"* | *"$nl"* | *'*'* | *'?'* | *'['*) echo "$0: invalid mode: $mode" >&2 exit 1;; esac shift;; -o) chowncmd="$chownprog $2" shift;; -s) stripcmd=$stripprog;; -t) is_target_a_directory=always dst_arg=$2 # Protect names problematic for 'test' and other utilities. case $dst_arg in -* | [=\(\)!]) dst_arg=./$dst_arg;; esac shift;; -T) is_target_a_directory=never;; --version) echo "$0 $scriptversion"; exit $?;; --) shift break;; -*) echo "$0: invalid option: $1" >&2 exit 1;; *) break;; esac shift done # We allow the use of options -d and -T together, by making -d # take the precedence; this is for compatibility with GNU install. if test -n "$dir_arg"; then if test -n "$dst_arg"; then echo "$0: target directory not allowed when installing a directory." >&2 exit 1 fi fi if test $# -ne 0 && test -z "$dir_arg$dst_arg"; then # When -d is used, all remaining arguments are directories to create. # When -t is used, the destination is already specified. # Otherwise, the last argument is the destination. Remove it from $@. for arg do if test -n "$dst_arg"; then # $@ is not empty: it contains at least $arg. set fnord "$@" "$dst_arg" shift # fnord fi shift # arg dst_arg=$arg # Protect names problematic for 'test' and other utilities. case $dst_arg in -* | [=\(\)!]) dst_arg=./$dst_arg;; esac done fi if test $# -eq 0; then if test -z "$dir_arg"; then echo "$0: no input file specified." >&2 exit 1 fi # It's OK to call 'install-sh -d' without argument. # This can happen when creating conditional directories. exit 0 fi if test -z "$dir_arg"; then if test $# -gt 1 || test "$is_target_a_directory" = always; then if test ! -d "$dst_arg"; then echo "$0: $dst_arg: Is not a directory." >&2 exit 1 fi fi fi if test -z "$dir_arg"; then do_exit='(exit $ret); exit $ret' trap "ret=129; $do_exit" 1 trap "ret=130; $do_exit" 2 trap "ret=141; $do_exit" 13 trap "ret=143; $do_exit" 15 # Set umask so as not to create temps with too-generous modes. # However, 'strip' requires both read and write access to temps. case $mode in # Optimize common cases. *644) cp_umask=133;; *755) cp_umask=22;; *[0-7]) if test -z "$stripcmd"; then u_plus_rw= else u_plus_rw='% 200' fi cp_umask=`expr '(' 777 - $mode % 1000 ')' $u_plus_rw`;; *) if test -z "$stripcmd"; then u_plus_rw= else u_plus_rw=,u+rw fi cp_umask=$mode$u_plus_rw;; esac fi for src do # Protect names problematic for 'test' and other utilities. case $src in -* | [=\(\)!]) src=./$src;; esac if test -n "$dir_arg"; then dst=$src dstdir=$dst test -d "$dstdir" dstdir_status=$? else # Waiting for this to be detected by the "$cpprog $src $dsttmp" command # might cause directories to be created, which would be especially bad # if $src (and thus $dsttmp) contains '*'. if test ! -f "$src" && test ! -d "$src"; then echo "$0: $src does not exist." >&2 exit 1 fi if test -z "$dst_arg"; then echo "$0: no destination specified." >&2 exit 1 fi dst=$dst_arg # If destination is a directory, append the input filename. if test -d "$dst"; then if test "$is_target_a_directory" = never; then echo "$0: $dst_arg: Is a directory" >&2 exit 1 fi dstdir=$dst dstbase=`basename "$src"` case $dst in */) dst=$dst$dstbase;; *) dst=$dst/$dstbase;; esac dstdir_status=0 else dstdir=`dirname "$dst"` test -d "$dstdir" dstdir_status=$? fi fi case $dstdir in */) dstdirslash=$dstdir;; *) dstdirslash=$dstdir/;; esac obsolete_mkdir_used=false if test $dstdir_status != 0; then case $posix_mkdir in '') # Create intermediate dirs using mode 755 as modified by the umask. # This is like FreeBSD 'install' as of 1997-10-28. umask=`umask` case $stripcmd.$umask in # Optimize common cases. *[2367][2367]) mkdir_umask=$umask;; .*0[02][02] | .[02][02] | .[02]) mkdir_umask=22;; *[0-7]) mkdir_umask=`expr $umask + 22 \ - $umask % 100 % 40 + $umask % 20 \ - $umask % 10 % 4 + $umask % 2 `;; *) mkdir_umask=$umask,go-w;; esac # With -d, create the new directory with the user-specified mode. # Otherwise, rely on $mkdir_umask. if test -n "$dir_arg"; then mkdir_mode=-m$mode else mkdir_mode= fi posix_mkdir=false case $umask in *[123567][0-7][0-7]) # POSIX mkdir -p sets u+wx bits regardless of umask, which # is incompatible with FreeBSD 'install' when (umask & 300) != 0. ;; *) # Note that $RANDOM variable is not portable (e.g. dash); Use it # here however when possible just to lower collision chance. tmpdir=${TMPDIR-/tmp}/ins$RANDOM-$$ trap 'ret=$?; rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" 2>/dev/null; exit $ret' 0 # Because "mkdir -p" follows existing symlinks and we likely work # directly in world-writeable /tmp, make sure that the '$tmpdir' # directory is successfully created first before we actually test # 'mkdir -p' feature. if (umask $mkdir_umask && $mkdirprog $mkdir_mode "$tmpdir" && exec $mkdirprog $mkdir_mode -p -- "$tmpdir/a/b") >/dev/null 2>&1 then if test -z "$dir_arg" || { # Check for POSIX incompatibilities with -m. # HP-UX 11.23 and IRIX 6.5 mkdir -m -p sets group- or # other-writable bit of parent directory when it shouldn't. # FreeBSD 6.1 mkdir -m -p sets mode of existing directory. test_tmpdir="$tmpdir/a" ls_ld_tmpdir=`ls -ld "$test_tmpdir"` case $ls_ld_tmpdir in d????-?r-*) different_mode=700;; d????-?--*) different_mode=755;; *) false;; esac && $mkdirprog -m$different_mode -p -- "$test_tmpdir" && { ls_ld_tmpdir_1=`ls -ld "$test_tmpdir"` test "$ls_ld_tmpdir" = "$ls_ld_tmpdir_1" } } then posix_mkdir=: fi rmdir "$tmpdir/a/b" "$tmpdir/a" "$tmpdir" else # Remove any dirs left behind by ancient mkdir implementations. rmdir ./$mkdir_mode ./-p ./-- "$tmpdir" 2>/dev/null fi trap '' 0;; esac;; esac if $posix_mkdir && ( umask $mkdir_umask && $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir" ) then : else # The umask is ridiculous, or mkdir does not conform to POSIX, # or it failed possibly due to a race condition. Create the # directory the slow way, step by step, checking for races as we go. case $dstdir in /*) prefix='/';; [-=\(\)!]*) prefix='./';; *) prefix='';; esac oIFS=$IFS IFS=/ set -f set fnord $dstdir shift set +f IFS=$oIFS prefixes= for d do test X"$d" = X && continue prefix=$prefix$d if test -d "$prefix"; then prefixes= else if $posix_mkdir; then (umask=$mkdir_umask && $doit_exec $mkdirprog $mkdir_mode -p -- "$dstdir") && break # Don't fail if two instances are running concurrently. test -d "$prefix" || exit 1 else case $prefix in *\'*) qprefix=`echo "$prefix" | sed "s/'/'\\\\\\\\''/g"`;; *) qprefix=$prefix;; esac prefixes="$prefixes '$qprefix'" fi fi prefix=$prefix/ done if test -n "$prefixes"; then # Don't fail if two instances are running concurrently. (umask $mkdir_umask && eval "\$doit_exec \$mkdirprog $prefixes") || test -d "$dstdir" || exit 1 obsolete_mkdir_used=true fi fi fi if test -n "$dir_arg"; then { test -z "$chowncmd" || $doit $chowncmd "$dst"; } && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dst"; } && { test "$obsolete_mkdir_used$chowncmd$chgrpcmd" = false || test -z "$chmodcmd" || $doit $chmodcmd $mode "$dst"; } || exit 1 else # Make a couple of temp file names in the proper directory. dsttmp=${dstdirslash}_inst.$$_ rmtmp=${dstdirslash}_rm.$$_ # Trap to clean up those temp files at exit. trap 'ret=$?; rm -f "$dsttmp" "$rmtmp" && exit $ret' 0 # Copy the file name to the temp name. (umask $cp_umask && $doit_exec $cpprog "$src" "$dsttmp") && # and set any options; do chmod last to preserve setuid bits. # # If any of these fail, we abort the whole thing. If we want to # ignore errors from any of these, just make sure not to ignore # errors from the above "$doit $cpprog $src $dsttmp" command. # { test -z "$chowncmd" || $doit $chowncmd "$dsttmp"; } && { test -z "$chgrpcmd" || $doit $chgrpcmd "$dsttmp"; } && { test -z "$stripcmd" || $doit $stripcmd "$dsttmp"; } && { test -z "$chmodcmd" || $doit $chmodcmd $mode "$dsttmp"; } && # If -C, don't bother to copy if it wouldn't change the file. if $copy_on_change && old=`LC_ALL=C ls -dlL "$dst" 2>/dev/null` && new=`LC_ALL=C ls -dlL "$dsttmp" 2>/dev/null` && set -f && set X $old && old=:$2:$4:$5:$6 && set X $new && new=:$2:$4:$5:$6 && set +f && test "$old" = "$new" && $cmpprog "$dst" "$dsttmp" >/dev/null 2>&1 then rm -f "$dsttmp" else # Rename the file to the real destination. $doit $mvcmd -f "$dsttmp" "$dst" 2>/dev/null || # The rename failed, perhaps because mv can't rename something else # to itself, or perhaps because mv is so ancient that it does not # support -f. { # Now remove or move aside any old file at destination location. # We try this two ways since rm can't unlink itself on some # systems and the destination file might be busy for other # reasons. In this case, the final cleanup might fail but the new # file should still install successfully. { test ! -f "$dst" || $doit $rmcmd -f "$dst" 2>/dev/null || { $doit $mvcmd -f "$dst" "$rmtmp" 2>/dev/null && { $doit $rmcmd -f "$rmtmp" 2>/dev/null; :; } } || { echo "$0: cannot unlink or rename $dst" >&2 (exit 1); exit 1 } } && # Now rename the file to the real destination. $doit $mvcmd "$dsttmp" "$dst" } fi || exit 1 trap '' 0 fi done # Local variables: # eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: dballe-8.6/fabfile.py0000644000175000017500000001276113554564112011512 00000000000000from __future__ import print_function from __future__ import absolute_import from __future__ import division from __future__ import unicode_literals from fabric.api import local, run, cd, env, hosts, put from io import BytesIO import git import re from six.moves import shlex_quote env.hosts = ["venti", "ventiquattro", "ventotto", "sette"] env.use_ssh_config = True def cmd(*args): return " ".join(shlex_quote(a) for a in args) @hosts("venti") def test_venti(): fedora_cxxflags = "-O2 -g -pipe -Wall -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong"\ " --param=ssp-buffer-size=4 -grecord-gcc-switches -m64 -mtune=generic" fedora_ldflags = "-Wl,-z,relro" repo = git.Repo() remote = repo.remote("venti") push_url = remote.config_reader.get("url") remote_dir = re.sub(r"^ssh://[^/]+", "", push_url) local(cmd("git", "push", "venti", "HEAD")) with cd(remote_dir): run(cmd("git", "checkout", "-B", "test_venti", repo.head.commit.hexsha)) run(cmd("git", "reset", "--hard")) run(cmd("git", "clean", "-fx")) run(cmd("autoreconf", "-if")) run(cmd("./configure", "--build=x86_64-redhat-linux-gnu", "--host=x86_64-redhat-linux-gnu", "--program-prefix=", "--prefix=/usr", "--exec-prefix=/usr", "--bindir=/usr/bin", "--sbindir=/usr/sbin", "--sysconfdir=/etc", "--datadir=/usr/share", "--includedir=/usr/include", "--libdir=/usr/lib64", "--libexecdir=/usr/libexec", "--localstatedir=/var", "--sharedstatedir=/var/lib", "--mandir=/usr/share/man", "--infodir=/usr/share/info", "CFLAGS=" + fedora_cxxflags, "CXXFLAGS=" + fedora_cxxflags, "LDFLAGS=" + fedora_ldflags)) run(cmd("make")) run(cmd("./run-check")) @hosts("ventiquattro") def test_ventiquattro(): fedora_cxxflags = "-O2 -g -pipe -Wall -Werror=format-security -Wp,-D_FORTIFY_SOURCE=2 -fexceptions -fstack-protector-strong"\ " --param=ssp-buffer-size=4 -grecord-gcc-switches -specs=/usr/lib/rpm/redhat/redhat-hardened-cc1 -m64 -mtune=generic" fedora_ldflags = "-Wl,-z,relro -specs=/usr/lib/rpm/redhat/redhat-hardened-ld" repo = git.Repo() remote = repo.remote("ventiquattro") push_url = remote.config_reader.get("url") remote_dir = re.sub(r"^ssh://[^/]+", "", push_url) local(cmd("git", "push", "ventiquattro", "HEAD")) with cd(remote_dir): run(cmd("git", "checkout", "-B", "test_ventiquattro", repo.head.commit.hexsha)) run(cmd("git", "reset", "--hard")) run(cmd("git", "clean", "-fx")) run(cmd("autoreconf", "-if")) run(cmd("./configure", "--build=x86_64-redhat-linux-gnu", "--host=x86_64-redhat-linux-gnu", "--program-prefix=", "--disable-dependency-tracking", "--prefix=/usr", "--exec-prefix=/usr", "--bindir=/usr/bin", "--sbindir=/usr/sbin", "--sysconfdir=/etc", "--datadir=/usr/share", "--includedir=/usr/include", "--libdir=/usr/lib64", "--libexecdir=/usr/libexec", "--localstatedir=/var", "--sharedstatedir=/var/lib", "--mandir=/usr/share/man", "--infodir=/usr/share/info", "CFLAGS=" + fedora_cxxflags, "CXXFLAGS=" + fedora_cxxflags, "LDFLAGS=" + fedora_ldflags)) run(cmd("make")) run(cmd("./run-check")) @hosts("ventotto") def test_ventotto(): repo = git.Repo() remote = repo.remote("ventotto") push_url = remote.config_reader.get("url") remote_dir = re.sub(r"^ssh://[^/]+", "", push_url) local(cmd("git", "push", "ventotto", "HEAD")) with cd(remote_dir): run(cmd("git", "checkout", "-B", "test_ventotto", repo.head.commit.hexsha)) run(cmd("git", "reset", "--hard")) run(cmd("git", "clean", "-fx")) run(cmd("autoreconf", "-if")) res = run(cmd("rpm", "--eval", "%configure FC=gfortran F90=gfortan F77=gfortran --enable-dballef --enable-dballe-python --enable-docs")) put(BytesIO(b"\n".join(res.stdout.splitlines())), "rpm-config") run(cmd("chmod", "0755", "rpm-config")) run(cmd("./rpm-config")) run(cmd("make", "-j2")) run(cmd("./run-check", "-j2")) @hosts("sette") def test_sette(): repo = git.Repo() remote = repo.remote("sette") push_url = remote.config_reader.get("url") remote_dir = re.sub(r"^ssh://[^/]+", "", push_url) local(cmd("git", "push", "sette", "HEAD")) with cd(remote_dir): run(cmd("git", "reset", "--hard")) run(cmd("git", "checkout", "-B", "test_sette", repo.head.commit.hexsha)) run(cmd("git", "reset", "--hard")) run(cmd("git", "clean", "-fx")) run(cmd("autoreconf", "-if")) res = run(cmd("rpm", "--eval", "%configure FC=gfortran F90=gfortan F77=gfortran --enable-dballef --enable-dballe-python --enable-docs")) put(BytesIO(b"\n".join(res.stdout.splitlines())), "rpm-config") run(cmd("chmod", "0755", "rpm-config")) run(cmd("./rpm-config")) run(cmd("make", "-j2")) run(cmd("./run-check", "-j2")) def test(): test_venti() test_ventiquattro() dballe-8.6/libdballef.pc.in0000644000175000017500000000031513554564112012551 00000000000000prefix=@prefix@ exec_prefix=@exec_prefix@ libdir=@libdir@ includedir=@includedir@ Name: libdballef Description: DB-All.e Fortran API Version: @VERSION@ Cflags: -I${includedir} Libs: -L${libdir} -ldballef dballe-8.6/dbaexport.10000644000175000017500000000214513572235553011621 00000000000000.\" DO NOT MODIFY THIS FILE! It was generated by help2man 1.47.8. .TH DBAEXPORT "1" "December 2019" "dbaexport 8.5" "User Commands" .SH NAME dbaexport \- export data from DB-All.e .SH DESCRIPTION usage: dbaexport [\-h] [\-\-verbose] [\-\-url url] [\-\-user name] [\-\-pass password] [\-\-outfile file] {csv,gnur,bufr,crex} ... .PP Export data from a DB\-All.e database. .SS "positional arguments:" .TP {csv,gnur,bufr,crex} output formats .TP csv export data as CSV .TP gnur export data as GNU R workspace .TP bufr export data as BUFR .TP crex export data as CREX .SS "optional arguments:" .TP \fB\-h\fR, \fB\-\-help\fR show this help message and exit .TP \fB\-\-verbose\fR verbose output .TP \fB\-\-url\fR url, \fB\-\-dsn\fR url URL\-like database definition, to use for connecting to the DB\-All.e database (can also be specified in the environment as DBA_DB) .TP \fB\-\-user\fR name username to use for connecting to the DB\-All.e database .TP \fB\-\-pass\fR password password to use for connecting to the DB\-All.e database .TP \fB\-\-outfile\fR file, \fB\-o\fR file output file. Default is standard output, if supported dballe-8.6/LICENSE0000644000175000017500000004321413554564112010552 00000000000000Copyright (C) 2005--2015 ARPA-SIM License: GNU GENERAL PUBLIC LICENSE Version 2, June 1991 Copyright (C) 1989, 1991 Free Software Foundation, Inc. 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed. Preamble The licenses for most software are designed to take away your freedom to share and change it. By contrast, the GNU General Public License is intended to guarantee your freedom to share and change free software--to make sure the software is free for all its users. This General Public License applies to most of the Free Software Foundation's software and to any other program whose authors commit to using it. (Some other Free Software Foundation software is covered by the GNU Library General Public License instead.) You can apply it to your programs, too. When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for this service if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs; and that you know you can do these things. To protect your rights, we need to make restrictions that forbid anyone to deny you these rights or to ask you to surrender the rights. These restrictions translate to certain responsibilities for you if you distribute copies of the software, or if you modify it. For example, if you distribute copies of such a program, whether gratis or for a fee, you must give the recipients all the rights that you have. You must make sure that they, too, receive or can get the source code. And you must show them these terms so they know their rights. We protect your rights with two steps: (1) copyright the software, and (2) offer you this license which gives you legal permission to copy, distribute and/or modify the software. Also, for each author's protection and ours, we want to make certain that everyone understands that there is no warranty for this free software. If the software is modified by someone else and passed on, we want its recipients to know that what they have is not the original, so that any problems introduced by others will not reflect on the original authors' reputations. Finally, any free program is threatened constantly by software patents. We wish to avoid the danger that redistributors of a free program will individually obtain patent licenses, in effect making the program proprietary. To prevent this, we have made it clear that any patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 0. This License applies to any program or other work which contains a notice placed by the copyright holder saying it may be distributed under the terms of this General Public License. The "Program", below, refers to any such program or work, and a "work based on the Program" means either the Program or any derivative work under copyright law: that is to say, a work containing the Program or a portion of it, either verbatim or with modifications and/or translated into another language. (Hereinafter, translation is included without limitation in the term "modification".) Each licensee is addressed as "you". Activities other than copying, distribution and modification are not covered by this License; they are outside its scope. The act of running the Program is not restricted, and the output from the Program is covered only if its contents constitute a work based on the Program (independent of having been made by running the Program). Whether that is true depends on what the Program does. 1. You may copy and distribute verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice and disclaimer of warranty; keep intact all the notices that refer to this License and to the absence of any warranty; and give any other recipients of the Program a copy of this License along with the Program. You may charge a fee for the physical act of transferring a copy, and you may at your option offer warranty protection in exchange for a fee. 2. You may modify your copy or copies of the Program or any portion of it, thus forming a work based on the Program, and copy and distribute such modifications or work under the terms of Section 1 above, provided that you also meet all of these conditions: a) You must cause the modified files to carry prominent notices stating that you changed the files and the date of any change. b) You must cause any work that you distribute or publish, that in whole or in part contains or is derived from the Program or any part thereof, to be licensed as a whole at no charge to all third parties under the terms of this License. c) If the modified program normally reads commands interactively when run, you must cause it, when started running for such interactive use in the most ordinary way, to print or display an announcement including an appropriate copyright notice and a notice that there is no warranty (or else, saying that you provide a warranty) and that users may redistribute the program under these conditions, and telling the user how to view a copy of this License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in themselves, then this License, and its terms, do not apply to those sections when you distribute them as separate works. But when you distribute the same sections as part of a whole which is a work based on the Program, the distribution of the whole must be on the terms of this License, whose permissions for other licensees extend to the entire whole, and thus to each and every part regardless of who wrote it. Thus, it is not the intent of this section to claim rights or contest your rights to work written entirely by you; rather, the intent is to exercise the right to control the distribution of derivative or collective works based on the Program. In addition, mere aggregation of another work not based on the Program with the Program (or with a work based on the Program) on a volume of a storage or distribution medium does not bring the other work under the scope of this License. 3. You may copy and distribute the Program (or a work based on it, under Section 2) in object code or executable form under the terms of Sections 1 and 2 above provided that you also do one of the following: a) Accompany it with the complete corresponding machine-readable source code, which must be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, b) Accompany it with a written offer, valid for at least three years, to give any third party, for a charge no more than your cost of physically performing source distribution, a complete machine-readable copy of the corresponding source code, to be distributed under the terms of Sections 1 and 2 above on a medium customarily used for software interchange; or, c) Accompany it with the information you received as to the offer to distribute corresponding source code. (This alternative is allowed only for noncommercial distribution and only if you received the program in object code or executable form with such an offer, in accord with Subsection b above.) The source code for a work means the preferred form of the work for making modifications to it. For an executable work, complete source code means all the source code for all modules it contains, plus any associated interface definition files, plus the scripts used to control compilation and installation of the executable. However, as a special exception, the source code distributed need not include anything that is normally distributed (in either source or binary form) with the major components (compiler, kernel, and so on) of the operating system on which the executable runs, unless that component itself accompanies the executable. If distribution of executable or object code is made by offering access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is void, and will automatically terminate your rights under this License. However, parties who have received copies, or rights, from you under this License will not have their licenses terminated so long as such parties remain in full compliance. 5. You are not required to accept this License, since you have not signed it. However, nothing else grants you permission to modify or distribute the Program or its derivative works. These actions are prohibited by law if you do not accept this License. Therefore, by modifying or distributing the Program (or any work based on the Program), you indicate your acceptance of this License to do so, and all its terms and conditions for copying, distributing or modifying the Program or works based on it. 6. Each time you redistribute the Program (or any work based on the Program), the recipient automatically receives a license from the original licensor to copy, distribute or modify the Program subject to these terms and conditions. You may not impose any further restrictions on the recipients' exercise of the rights granted herein. You are not responsible for enforcing compliance by third parties to this License. 7. If, as a consequence of a court judgment or allegation of patent infringement or for any other reason (not limited to patent issues), conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot distribute so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may not distribute the Program at all. For example, if a patent license would not permit royalty-free redistribution of the Program by all those who receive copies directly or indirectly through you, then the only way you could satisfy both it and this License would be to refrain entirely from distribution of the Program. If any portion of this section is held invalid or unenforceable under any particular circumstance, the balance of the section is intended to apply and the section as a whole is intended to apply in other circumstances. It is not the purpose of this section to induce you to infringe any patents or other property right claims or to contest validity of any such claims; this section has the sole purpose of protecting the integrity of the free software distribution system, which is implemented by public license practices. Many people have made generous contributions to the wide range of software distributed through that system in reliance on consistent application of that system; it is up to the author/donor to decide if he or she is willing to distribute software through any other system and a licensee cannot impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License may add an explicit geographical distribution limitation excluding those countries, so that distribution is permitted only in or among countries not thus excluded. In such case, this License incorporates the limitation as if written in the body of this License. 9. The Free Software Foundation may publish revised and/or new versions of the General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns. Each version is given a distinguishing version number. If the Program specifies a version number of this License which applies to it and "any later version", you have the option of following the terms and conditions either of that version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of this License, you may choose any version ever published by the Free Software Foundation. 10. If you wish to incorporate parts of the Program into other free programs whose distribution conditions are different, write to the author to ask for permission. For software which is copyrighted by the Free Software Foundation, write to the Free Software Foundation; we sometimes make exceptions for this. Our decision will be guided by the two goals of preserving the free status of all derivatives of our free software and of promoting the sharing and reuse of software generally. NO WARRANTY 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION. 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms. To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively convey the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found. Copyright (C) This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA Also add information on how to contact you by electronic and paper mail. If the program is interactive, make it output a short notice like this when it starts in an interactive mode: Gnomovision version 69, Copyright (C) year name of author Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. This is free software, and you are welcome to redistribute it under certain conditions; type `show c' for details. The hypothetical commands `show w' and `show c' should show the appropriate parts of the General Public License. Of course, the commands you use may be called something other than `show w' and `show c'; they could even be mouse-clicks or menu items--whatever suits your program. You should also get your employer (if you work as a programmer) or your school, if any, to sign a "copyright disclaimer" for the program, if necessary. Here is a sample; alter the names: Yoyodyne, Inc., hereby disclaims all copyright interest in the program `Gnomovision' (which makes passes at compilers) written by James Hacker. , 1 April 1989 Ty Coon, President of Vice This General Public License does not permit incorporating your program into proprietary programs. If your program is a subroutine library, you may consider it more useful to permit linking proprietary applications with the library. If this is what you want to do, use the GNU Library General Public License instead of this License. dballe-8.6/get_line_no0000755000175000017500000000100513554564112011745 00000000000000#!/usr/bin/perl -w # Convert a file offset into a line number use strict; use warnings; if (!@ARGV) { print qq{Usage: $0 file offset Prints the line number of the given offset in the file. }; exit 0; } my $file = $ARGV[0]; my $num = $ARGV[1]; open(IN, $file) or die "Cannot open $file: $!"; my $ofs = 0; while (<>) { my $len = length($_); if ($ofs + $len > $num) { printf "%d:%d\n", $., $num - $ofs; exit 0; } $ofs += $len; } close(IN); print STDERR "Offset is past the end of the file\n"; exit 1; dballe-8.6/compile0000755000175000017500000001632713602151750011122 00000000000000#! /bin/sh # Wrapper for compilers which do not understand '-c -o'. scriptversion=2018-03-07.03; # UTC # Copyright (C) 1999-2018 Free Software Foundation, Inc. # Written by Tom Tromey . # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. # This file is maintained in Automake, please report # bugs to or send patches to # . nl=' ' # We need space, tab and new line, in precisely that order. Quoting is # there to prevent tools from complaining about whitespace usage. IFS=" "" $nl" file_conv= # func_file_conv build_file lazy # Convert a $build file to $host form and store it in $file # Currently only supports Windows hosts. If the determined conversion # type is listed in (the comma separated) LAZY, no conversion will # take place. func_file_conv () { file=$1 case $file in / | /[!/]*) # absolute file, and not a UNC file if test -z "$file_conv"; then # lazily determine how to convert abs files case `uname -s` in MINGW*) file_conv=mingw ;; CYGWIN*) file_conv=cygwin ;; *) file_conv=wine ;; esac fi case $file_conv/,$2, in *,$file_conv,*) ;; mingw/*) file=`cmd //C echo "$file " | sed -e 's/"\(.*\) " *$/\1/'` ;; cygwin/*) file=`cygpath -m "$file" || echo "$file"` ;; wine/*) file=`winepath -w "$file" || echo "$file"` ;; esac ;; esac } # func_cl_dashL linkdir # Make cl look for libraries in LINKDIR func_cl_dashL () { func_file_conv "$1" if test -z "$lib_path"; then lib_path=$file else lib_path="$lib_path;$file" fi linker_opts="$linker_opts -LIBPATH:$file" } # func_cl_dashl library # Do a library search-path lookup for cl func_cl_dashl () { lib=$1 found=no save_IFS=$IFS IFS=';' for dir in $lib_path $LIB do IFS=$save_IFS if $shared && test -f "$dir/$lib.dll.lib"; then found=yes lib=$dir/$lib.dll.lib break fi if test -f "$dir/$lib.lib"; then found=yes lib=$dir/$lib.lib break fi if test -f "$dir/lib$lib.a"; then found=yes lib=$dir/lib$lib.a break fi done IFS=$save_IFS if test "$found" != yes; then lib=$lib.lib fi } # func_cl_wrapper cl arg... # Adjust compile command to suit cl func_cl_wrapper () { # Assume a capable shell lib_path= shared=: linker_opts= for arg do if test -n "$eat"; then eat= else case $1 in -o) # configure might choose to run compile as 'compile cc -o foo foo.c'. eat=1 case $2 in *.o | *.[oO][bB][jJ]) func_file_conv "$2" set x "$@" -Fo"$file" shift ;; *) func_file_conv "$2" set x "$@" -Fe"$file" shift ;; esac ;; -I) eat=1 func_file_conv "$2" mingw set x "$@" -I"$file" shift ;; -I*) func_file_conv "${1#-I}" mingw set x "$@" -I"$file" shift ;; -l) eat=1 func_cl_dashl "$2" set x "$@" "$lib" shift ;; -l*) func_cl_dashl "${1#-l}" set x "$@" "$lib" shift ;; -L) eat=1 func_cl_dashL "$2" ;; -L*) func_cl_dashL "${1#-L}" ;; -static) shared=false ;; -Wl,*) arg=${1#-Wl,} save_ifs="$IFS"; IFS=',' for flag in $arg; do IFS="$save_ifs" linker_opts="$linker_opts $flag" done IFS="$save_ifs" ;; -Xlinker) eat=1 linker_opts="$linker_opts $2" ;; -*) set x "$@" "$1" shift ;; *.cc | *.CC | *.cxx | *.CXX | *.[cC]++) func_file_conv "$1" set x "$@" -Tp"$file" shift ;; *.c | *.cpp | *.CPP | *.lib | *.LIB | *.Lib | *.OBJ | *.obj | *.[oO]) func_file_conv "$1" mingw set x "$@" "$file" shift ;; *) set x "$@" "$1" shift ;; esac fi shift done if test -n "$linker_opts"; then linker_opts="-link$linker_opts" fi exec "$@" $linker_opts exit 1 } eat= case $1 in '') echo "$0: No command. Try '$0 --help' for more information." 1>&2 exit 1; ;; -h | --h*) cat <<\EOF Usage: compile [--help] [--version] PROGRAM [ARGS] Wrapper for compilers which do not understand '-c -o'. Remove '-o dest.o' from ARGS, run PROGRAM with the remaining arguments, and rename the output as expected. If you are trying to build a whole package this is not the right script to run: please start by reading the file 'INSTALL'. Report bugs to . EOF exit $? ;; -v | --v*) echo "compile $scriptversion" exit $? ;; cl | *[/\\]cl | cl.exe | *[/\\]cl.exe | \ icl | *[/\\]icl | icl.exe | *[/\\]icl.exe ) func_cl_wrapper "$@" # Doesn't return... ;; esac ofile= cfile= for arg do if test -n "$eat"; then eat= else case $1 in -o) # configure might choose to run compile as 'compile cc -o foo foo.c'. # So we strip '-o arg' only if arg is an object. eat=1 case $2 in *.o | *.obj) ofile=$2 ;; *) set x "$@" -o "$2" shift ;; esac ;; *.c) cfile=$1 set x "$@" "$1" shift ;; *) set x "$@" "$1" shift ;; esac fi shift done if test -z "$ofile" || test -z "$cfile"; then # If no '-o' option was seen then we might have been invoked from a # pattern rule where we don't need one. That is ok -- this is a # normal compilation that the losing compiler can handle. If no # '.c' file was seen then we are probably linking. That is also # ok. exec "$@" fi # Name of file we expect compiler to create. cofile=`echo "$cfile" | sed 's|^.*[\\/]||; s|^[a-zA-Z]:||; s/\.c$/.o/'` # Create the lock directory. # Note: use '[/\\:.-]' here to ensure that we don't use the same name # that we are using for the .o file. Also, base the name on the expected # object file name, since that is what matters with a parallel build. lockdir=`echo "$cofile" | sed -e 's|[/\\:.-]|_|g'`.d while true; do if mkdir "$lockdir" >/dev/null 2>&1; then break fi sleep 1 done # FIXME: race condition here if user kills between mkdir and trap. trap "rmdir '$lockdir'; exit 1" 1 2 15 # Run the compile. "$@" ret=$? if test -f "$cofile"; then test "$cofile" = "$ofile" || mv "$cofile" "$ofile" elif test -f "${cofile}bj"; then test "${cofile}bj" = "$ofile" || mv "${cofile}bj" "$ofile" fi rmdir "$lockdir" exit $ret # Local Variables: # mode: shell-script # sh-indentation: 2 # eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: dballe-8.6/bench/0000755000175000017500000000000013602152022010663 500000000000000dballe-8.6/bench/query.cc0000644000175000017500000000463313554564112012302 00000000000000#include #include #include #include #include #include struct BenchmarkQuery : public dballe::benchmark::Task { std::shared_ptr db; const char* m_name; const char* m_pathname; unsigned months; unsigned hours; unsigned minutes; BenchmarkQuery(const char* name, const char* pathname, unsigned months=12, unsigned hours=24, unsigned minutes=1) : m_name(name), m_pathname(pathname), months(months), hours(hours), minutes(minutes) { auto options = dballe::DBConnectOptions::test_create(); db = dballe::db::DB::downcast(dballe::DB::connect(*options)); } const char* name() const override { return m_name; } void setup() override { db->reset(); dballe::benchmark::Messages messages; messages.load(m_pathname); // Multiply messages by changing their datetime size_t size = messages.size(); for (unsigned year = 2016; year < 2018; ++year) for (unsigned month = 1; month <= months; ++month) for (unsigned hour = 0; hour < hours; ++hour) for (unsigned minute = 0; minute < minutes; ++minute) messages.duplicate(size, dballe::Datetime(year, month, 1, hour, minute)); auto tr = db->transaction(); for (const auto& msgs: messages) tr->import_messages(msgs); tr->commit(); } void run_once() override { auto tr = std::dynamic_pointer_cast(db->transaction()); dballe::core::Query query; auto cur = tr->query_data(query); while (cur->next()) ; tr->commit(); } void teardown() override { db->remove_all(); } }; int main(int argc, const char* argv[]) { using namespace dballe::benchmark; dballe::benchmark::Task* tasks[] = { new BenchmarkQuery("synop", "extra/bufr/synop-rad1.bufr", 1, 24), new BenchmarkQuery("temp", "extra/bufr/temp-huge.bufr", 1, 1), new BenchmarkQuery("acars", "extra/bufr/gts-acars2.bufr", 12, 24, 10), }; Benchmark benchmark; dballe::benchmark::Whitelist whitelist(argc, argv); for (auto task: tasks) if (whitelist.has(task->name())) benchmark.timeit(*task, 20); benchmark.print_timings(); return 0; } dballe-8.6/bench/Makefile.am0000644000175000017500000000070413554564112012655 00000000000000## Process this file with automake to produce Makefile.in DBALLELIBS = ../dballe/libdballe.la AM_CPPFLAGS = -I$(top_srcdir) $(WREPORT_CFLAGS) $(LUA_CFLAGS) -Werror if FILE_OFFSET_BITS_64 AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64 endif noinst_PROGRAMS = import query import_SOURCES = import.cc import_LDFLAGS = $(DBALLELIBS) import_DEPENDENCIES = $(DBALLELIBS) query_SOURCES = query.cc query_LDFLAGS = $(DBALLELIBS) query_DEPENDENCIES = $(DBALLELIBS) dballe-8.6/bench/import.cc0000644000175000017500000000410613554564112012442 00000000000000#include #include #include #include #include struct BenchmarkImport : public dballe::benchmark::Task { dballe::benchmark::Messages messages; std::shared_ptr db; const char* m_name; const char* m_pathname; unsigned hours; unsigned minutes; BenchmarkImport(const char* name, const char* pathname, unsigned hours=24, unsigned minutes=1) : m_name(name), m_pathname(pathname), hours(hours), minutes(minutes) { auto options = dballe::DBConnectOptions::test_create(); db = dballe::db::DB::downcast(dballe::DB::connect(*options)); } const char* name() const override { return m_name; } void setup() override { db->reset(); messages.load(m_pathname); // Multiply messages by changing their datetime size_t size = messages.size(); for (unsigned year = 2016; year < 2018; ++year) for (unsigned month = 1; month <= 12; ++month) for (unsigned hour = 0; hour < hours; ++hour) for (unsigned minute = 0; minute < minutes; ++minute) messages.duplicate(size, dballe::Datetime(year, month, 1, hour, minute)); } void run_once() override { auto tr = db->transaction(); for (const auto& msgs: messages) tr->import_messages(msgs); tr->commit(); } void teardown() override { db->remove_all(); } }; int main(int argc, const char* argv[]) { using namespace dballe::benchmark; dballe::benchmark::Task* tasks[] = { new BenchmarkImport("synop", "extra/bufr/synop-rad1.bufr"), new BenchmarkImport("temp", "extra/bufr/temp-huge.bufr", 2), new BenchmarkImport("acars", "extra/bufr/gts-acars2.bufr", 24, 15), }; Benchmark benchmark; dballe::benchmark::Whitelist whitelist(argc, argv); for (auto task: tasks) if (whitelist.has(task->name())) benchmark.timeit(*task); benchmark.print_timings(); return 0; } dballe-8.6/bench/Makefile.in0000644000175000017500000005027013602151750012663 00000000000000# Makefile.in generated by automake 1.16.1 from Makefile.am. # @configure_input@ # Copyright (C) 1994-2018 Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY, to the extent permitted by law; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR A # PARTICULAR PURPOSE. @SET_MAKE@ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ false; \ elif test -n '$(MAKE_HOST)'; then \ true; \ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ true; \ else \ false; \ fi; \ } am__make_running_with_option = \ case $${target_option-} in \ ?) ;; \ *) echo "am__make_running_with_option: internal error: invalid" \ "target option '$${target_option-}' specified" >&2; \ exit 1;; \ esac; \ has_opt=no; \ sane_makeflags=$$MAKEFLAGS; \ if $(am__is_gnu_make); then \ sane_makeflags=$$MFLAGS; \ else \ case $$MAKEFLAGS in \ *\\[\ \ ]*) \ bs=\\; \ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ esac; \ fi; \ skip_next=no; \ strip_trailopt () \ { \ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ }; \ for flg in $$sane_makeflags; do \ test $$skip_next = yes && { skip_next=no; continue; }; \ case $$flg in \ *=*|--*) continue;; \ -*I) strip_trailopt 'I'; skip_next=yes;; \ -*I?*) strip_trailopt 'I';; \ -*O) strip_trailopt 'O'; skip_next=yes;; \ -*O?*) strip_trailopt 'O';; \ -*l) strip_trailopt 'l'; skip_next=yes;; \ -*l?*) strip_trailopt 'l';; \ -[dEDm]) skip_next=yes;; \ -[JT]) skip_next=yes;; \ esac; \ case $$flg in \ *$$target_option*) has_opt=yes; break;; \ esac; \ done; \ test $$has_opt = yes am__make_dryrun = (target_option=n; $(am__make_running_with_option)) am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) pkgdatadir = $(datadir)/@PACKAGE@ pkgincludedir = $(includedir)/@PACKAGE@ pkglibdir = $(libdir)/@PACKAGE@ pkglibexecdir = $(libexecdir)/@PACKAGE@ am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd install_sh_DATA = $(install_sh) -c -m 644 install_sh_PROGRAM = $(install_sh) -c install_sh_SCRIPT = $(install_sh) -c INSTALL_HEADER = $(INSTALL_DATA) transform = $(program_transform_name) NORMAL_INSTALL = : PRE_INSTALL = : POST_INSTALL = : NORMAL_UNINSTALL = : PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ @FILE_OFFSET_BITS_64_TRUE@am__append_1 = -D_FILE_OFFSET_BITS=64 noinst_PROGRAMS = import$(EXEEXT) query$(EXEEXT) subdir = bench ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/ax_append_flag.m4 \ $(top_srcdir)/m4/ax_cflags_warn_all.m4 \ $(top_srcdir)/m4/ax_python_module.m4 \ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ $(top_srcdir)/m4/lt~obsolete.m4 \ $(top_srcdir)/m4/m4_ax_cxx_compile_stdcxx_11.m4 \ $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/m4/python.m4 \ $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) DIST_COMMON = $(srcdir)/Makefile.am $(am__DIST_COMMON) mkinstalldirs = $(install_sh) -d CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = PROGRAMS = $(noinst_PROGRAMS) am_import_OBJECTS = import.$(OBJEXT) import_OBJECTS = $(am_import_OBJECTS) import_LDADD = $(LDADD) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent am__v_lt_1 = import_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(import_LDFLAGS) $(LDFLAGS) -o $@ am_query_OBJECTS = query.$(OBJEXT) query_OBJECTS = $(am_query_OBJECTS) query_LDADD = $(LDADD) query_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(query_LDFLAGS) $(LDFLAGS) -o $@ AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) am__v_P_0 = false am__v_P_1 = : AM_V_GEN = $(am__v_GEN_@AM_V@) am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) am__v_GEN_0 = @echo " GEN " $@; am__v_GEN_1 = AM_V_at = $(am__v_at_@AM_V@) am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) am__v_at_0 = @ am__v_at_1 = DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/depcomp am__maybe_remake_depfiles = depfiles am__depfiles_remade = ./$(DEPDIR)/import.Po ./$(DEPDIR)/query.Po am__mv = mv -f CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ $(AM_CXXFLAGS) $(CXXFLAGS) AM_V_CXX = $(am__v_CXX_@AM_V@) am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) am__v_CXX_0 = @echo " CXX " $@; am__v_CXX_1 = CXXLD = $(CXX) CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) am__v_CXXLD_0 = @echo " CXXLD " $@; am__v_CXXLD_1 = SOURCES = $(import_SOURCES) $(query_SOURCES) DIST_SOURCES = $(import_SOURCES) $(query_SOURCES) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) # Read a list of newline-separated strings from the standard input, # and print each of them once, without duplicates. Input order is # *not* preserved. am__uniquify_input = $(AWK) '\ BEGIN { nonempty = 0; } \ { items[$$0] = 1; nonempty = 1; } \ END { if (nonempty) { for (i in items) print i; }; } \ ' # Make sure the list of sources is unique. This is necessary because, # e.g., the same source file might be shared among _SOURCES variables # for different programs/libraries. am__define_uniq_tagged_files = \ list='$(am__tagged_files)'; \ unique=`for i in $$list; do \ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ done | $(am__uniquify_input)` ETAGS = etags CTAGS = ctags am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ AMTAR = @AMTAR@ AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ AR = @AR@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ CC = @CC@ CCDEPMODE = @CCDEPMODE@ CFLAGS = @CFLAGS@ CPP = @CPP@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXCPP = @CXXCPP@ CXXDEPMODE = @CXXDEPMODE@ CXXFLAGS = @CXXFLAGS@ CYGPATH_W = @CYGPATH_W@ DEFS = @DEFS@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DOXYGEN = @DOXYGEN@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ EGREP = @EGREP@ EXEEXT = @EXEEXT@ FC = @FC@ FCFLAGS = @FCFLAGS@ FGREP = @FGREP@ GREP = @GREP@ HAVE_CXX11 = @HAVE_CXX11@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ LD = @LD@ LDFLAGS = @LDFLAGS@ LIBDBALLEF_VERSION_INFO = @LIBDBALLEF_VERSION_INFO@ LIBDBALLE_VERSION_INFO = @LIBDBALLE_VERSION_INFO@ LIBOBJS = @LIBOBJS@ LIBPQ_CFLAGS = @LIBPQ_CFLAGS@ LIBPQ_LIBS = @LIBPQ_LIBS@ LIBS = @LIBS@ LIBTOOL = @LIBTOOL@ LIPO = @LIPO@ LN_S = @LN_S@ LTLIBOBJS = @LTLIBOBJS@ LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ MAKEINFO = @MAKEINFO@ MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MYSQL_CFLAGS = @MYSQL_CFLAGS@ MYSQL_LIBS = @MYSQL_LIBS@ NM = @NM@ NMEDIT = @NMEDIT@ OBJDUMP = @OBJDUMP@ OBJEXT = @OBJEXT@ OTOOL = @OTOOL@ OTOOL64 = @OTOOL64@ PACKAGE = @PACKAGE@ PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_STRING = @PACKAGE_STRING@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ PKGCONFIG_CFLAGS = @PKGCONFIG_CFLAGS@ PKGCONFIG_LIBS = @PKGCONFIG_LIBS@ PKGCONFIG_REQUIRES = @PKGCONFIG_REQUIRES@ PKG_CONFIG = @PKG_CONFIG@ POPT_CFLAGS = @POPT_CFLAGS@ POPT_LIBS = @POPT_LIBS@ PYTHON = @PYTHON@ PYTHON_CFLAGS = @PYTHON_CFLAGS@ PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ PYTHON_MAJOR_VERSION = @PYTHON_MAJOR_VERSION@ PYTHON_PLATFORM = @PYTHON_PLATFORM@ PYTHON_PREFIX = @PYTHON_PREFIX@ PYTHON_VERSION = @PYTHON_VERSION@ RANLIB = @RANLIB@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SPHINX_BUILD = @SPHINX_BUILD@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@ SQLITE3_LIBS = @SQLITE3_LIBS@ STRIP = @STRIP@ VERSION = @VERSION@ WREPORT_CFLAGS = @WREPORT_CFLAGS@ WREPORT_DOXYGEN_DIR = @WREPORT_DOXYGEN_DIR@ WREPORT_DOXYGEN_TAGFILE = @WREPORT_DOXYGEN_TAGFILE@ WREPORT_LIBS = @WREPORT_LIBS@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ ac_ct_FC = @ac_ct_FC@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ am__quote = @am__quote@ am__tar = @am__tar@ am__untar = @am__untar@ bindir = @bindir@ build = @build@ build_alias = @build_alias@ build_cpu = @build_cpu@ build_os = @build_os@ build_vendor = @build_vendor@ builddir = @builddir@ confdir = @confdir@ datadir = @datadir@ datarootdir = @datarootdir@ docdir = @docdir@ dvidir = @dvidir@ exec_prefix = @exec_prefix@ have_doxygen = @have_doxygen@ have_gperf = @have_gperf@ host = @host@ host_alias = @host_alias@ host_cpu = @host_cpu@ host_os = @host_os@ host_vendor = @host_vendor@ htmldir = @htmldir@ includedir = @includedir@ infodir = @infodir@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ mysql_config = @mysql_config@ oldincludedir = @oldincludedir@ pdfdir = @pdfdir@ pkgpyexecdir = @pkgpyexecdir@ pkgpythondir = @pkgpythondir@ prefix = @prefix@ program_transform_name = @program_transform_name@ psdir = @psdir@ pyexecdir = @pyexecdir@ pythondir = @pythondir@ runstatedir = @runstatedir@ sbindir = @sbindir@ sharedstatedir = @sharedstatedir@ srcdir = @srcdir@ sysconfdir = @sysconfdir@ tabledir = @tabledir@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ DBALLELIBS = ../dballe/libdballe.la AM_CPPFLAGS = -I$(top_srcdir) $(WREPORT_CFLAGS) $(LUA_CFLAGS) -Werror \ $(am__append_1) import_SOURCES = import.cc import_LDFLAGS = $(DBALLELIBS) import_DEPENDENCIES = $(DBALLELIBS) query_SOURCES = query.cc query_LDFLAGS = $(DBALLELIBS) query_DEPENDENCIES = $(DBALLELIBS) all: all-am .SUFFIXES: .SUFFIXES: .cc .lo .o .obj $(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) @for dep in $?; do \ case '$(am__configure_deps)' in \ *$$dep*) \ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ && { if test -f $@; then exit 0; else break; fi; }; \ exit 1;; \ esac; \ done; \ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign bench/Makefile'; \ $(am__cd) $(top_srcdir) && \ $(AUTOMAKE) --foreign bench/Makefile Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status @case '$?' in \ *config.status*) \ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ *) \ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ esac; $(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(top_srcdir)/configure: $(am__configure_deps) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(ACLOCAL_M4): $(am__aclocal_m4_deps) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(am__aclocal_m4_deps): clean-noinstPROGRAMS: @list='$(noinst_PROGRAMS)'; test -n "$$list" || exit 0; \ echo " rm -f" $$list; \ rm -f $$list || exit $$?; \ test -n "$(EXEEXT)" || exit 0; \ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ echo " rm -f" $$list; \ rm -f $$list import$(EXEEXT): $(import_OBJECTS) $(import_DEPENDENCIES) $(EXTRA_import_DEPENDENCIES) @rm -f import$(EXEEXT) $(AM_V_CXXLD)$(import_LINK) $(import_OBJECTS) $(import_LDADD) $(LIBS) query$(EXEEXT): $(query_OBJECTS) $(query_DEPENDENCIES) $(EXTRA_query_DEPENDENCIES) @rm -f query$(EXEEXT) $(AM_V_CXXLD)$(query_LINK) $(query_OBJECTS) $(query_LDADD) $(LIBS) mostlyclean-compile: -rm -f *.$(OBJEXT) distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/import.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/query.Po@am__quote@ # am--include-marker $(am__depfiles_remade): @$(MKDIR_P) $(@D) @echo '# dummy' >$@-t && $(am__mv) $@-t $@ am--depfiles: $(am__depfiles_remade) .cc.o: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ @am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< .cc.obj: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ @am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` .cc.lo: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ @am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< mostlyclean-libtool: -rm -f *.lo clean-libtool: -rm -rf .libs _libs ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique tags: tags-am TAGS: tags tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) set x; \ here=`pwd`; \ $(am__define_uniq_tagged_files); \ shift; \ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ test -n "$$unique" || unique=$$empty_fix; \ if test $$# -gt 0; then \ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ "$$@" $$unique; \ else \ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ $$unique; \ fi; \ fi ctags: ctags-am CTAGS: ctags ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) $(am__define_uniq_tagged_files); \ test -z "$(CTAGS_ARGS)$$unique" \ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ $$unique GTAGS: here=`$(am__cd) $(top_builddir) && pwd` \ && $(am__cd) $(top_srcdir) \ && gtags -i $(GTAGS_ARGS) "$$here" cscopelist: cscopelist-am cscopelist-am: $(am__tagged_files) list='$(am__tagged_files)'; \ case "$(srcdir)" in \ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ *) sdir=$(subdir)/$(srcdir) ;; \ esac; \ for i in $$list; do \ if test -f "$$i"; then \ echo "$(subdir)/$$i"; \ else \ echo "$$sdir/$$i"; \ fi; \ done >> $(top_builddir)/cscope.files distclean-tags: -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags distdir: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) distdir-am distdir-am: $(DISTFILES) @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ list='$(DISTFILES)'; \ dist_files=`for file in $$list; do echo $$file; done | \ sed -e "s|^$$srcdirstrip/||;t" \ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ case $$dist_files in \ */*) $(MKDIR_P) `echo "$$dist_files" | \ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ sort -u` ;; \ esac; \ for file in $$dist_files; do \ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ if test -d $$d/$$file; then \ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ if test -d "$(distdir)/$$file"; then \ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ fi; \ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ fi; \ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ else \ test -f "$(distdir)/$$file" \ || cp -p $$d/$$file "$(distdir)/$$file" \ || exit 1; \ fi; \ done check-am: all-am check: check-am all-am: Makefile $(PROGRAMS) installdirs: install: install-am install-exec: install-exec-am install-data: install-data-am uninstall: uninstall-am install-am: all-am @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am installcheck: installcheck-am install-strip: if test -z '$(STRIP)'; then \ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ install; \ else \ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ fi mostlyclean-generic: clean-generic: distclean-generic: -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." clean: clean-am clean-am: clean-generic clean-libtool clean-noinstPROGRAMS \ mostlyclean-am distclean: distclean-am -rm -f ./$(DEPDIR)/import.Po -rm -f ./$(DEPDIR)/query.Po -rm -f Makefile distclean-am: clean-am distclean-compile distclean-generic \ distclean-tags dvi: dvi-am dvi-am: html: html-am html-am: info: info-am info-am: install-data-am: install-dvi: install-dvi-am install-dvi-am: install-exec-am: install-html: install-html-am install-html-am: install-info: install-info-am install-info-am: install-man: install-pdf: install-pdf-am install-pdf-am: install-ps: install-ps-am install-ps-am: installcheck-am: maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/import.Po -rm -f ./$(DEPDIR)/query.Po -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic mostlyclean: mostlyclean-am mostlyclean-am: mostlyclean-compile mostlyclean-generic \ mostlyclean-libtool pdf: pdf-am pdf-am: ps: ps-am ps-am: uninstall-am: .MAKE: install-am install-strip .PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ clean-generic clean-libtool clean-noinstPROGRAMS cscopelist-am \ ctags ctags-am distclean distclean-compile distclean-generic \ distclean-libtool distclean-tags distdir dvi dvi-am html \ html-am info info-am install install-am install-data \ install-data-am install-dvi install-dvi-am install-exec \ install-exec-am install-html install-html-am install-info \ install-info-am install-man install-pdf install-pdf-am \ install-ps install-ps-am install-strip installcheck \ installcheck-am installdirs maintainer-clean \ maintainer-clean-generic mostlyclean mostlyclean-compile \ mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ tags tags-am uninstall uninstall-am .PRECIOUS: Makefile # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: dballe-8.6/show_code_notes0000755000175000017500000000006313554564112012650 00000000000000#!/bin/sh find . | xargs grep 'FIXME\|TODO' | less dballe-8.6/depcomp0000755000175000017500000005602013602151750011113 00000000000000#! /bin/sh # depcomp - compile a program generating dependencies as side-effects scriptversion=2018-03-07.03; # UTC # Copyright (C) 1999-2018 Free Software Foundation, Inc. # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2, or (at your option) # any later version. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # You should have received a copy of the GNU General Public License # along with this program. If not, see . # As a special exception to the GNU General Public License, if you # distribute this file as part of a program that contains a # configuration script generated by Autoconf, you may include it under # the same distribution terms that you use for the rest of that program. # Originally written by Alexandre Oliva . case $1 in '') echo "$0: No command. Try '$0 --help' for more information." 1>&2 exit 1; ;; -h | --h*) cat <<\EOF Usage: depcomp [--help] [--version] PROGRAM [ARGS] Run PROGRAMS ARGS to compile a file, generating dependencies as side-effects. Environment variables: depmode Dependency tracking mode. source Source file read by 'PROGRAMS ARGS'. object Object file output by 'PROGRAMS ARGS'. DEPDIR directory where to store dependencies. depfile Dependency file to output. tmpdepfile Temporary file to use when outputting dependencies. libtool Whether libtool is used (yes/no). Report bugs to . EOF exit $? ;; -v | --v*) echo "depcomp $scriptversion" exit $? ;; esac # Get the directory component of the given path, and save it in the # global variables '$dir'. Note that this directory component will # be either empty or ending with a '/' character. This is deliberate. set_dir_from () { case $1 in */*) dir=`echo "$1" | sed -e 's|/[^/]*$|/|'`;; *) dir=;; esac } # Get the suffix-stripped basename of the given path, and save it the # global variable '$base'. set_base_from () { base=`echo "$1" | sed -e 's|^.*/||' -e 's/\.[^.]*$//'` } # If no dependency file was actually created by the compiler invocation, # we still have to create a dummy depfile, to avoid errors with the # Makefile "include basename.Plo" scheme. make_dummy_depfile () { echo "#dummy" > "$depfile" } # Factor out some common post-processing of the generated depfile. # Requires the auxiliary global variable '$tmpdepfile' to be set. aix_post_process_depfile () { # If the compiler actually managed to produce a dependency file, # post-process it. if test -f "$tmpdepfile"; then # Each line is of the form 'foo.o: dependency.h'. # Do two passes, one to just change these to # $object: dependency.h # and one to simply output # dependency.h: # which is needed to avoid the deleted-header problem. { sed -e "s,^.*\.[$lower]*:,$object:," < "$tmpdepfile" sed -e "s,^.*\.[$lower]*:[$tab ]*,," -e 's,$,:,' < "$tmpdepfile" } > "$depfile" rm -f "$tmpdepfile" else make_dummy_depfile fi } # A tabulation character. tab=' ' # A newline character. nl=' ' # Character ranges might be problematic outside the C locale. # These definitions help. upper=ABCDEFGHIJKLMNOPQRSTUVWXYZ lower=abcdefghijklmnopqrstuvwxyz digits=0123456789 alpha=${upper}${lower} if test -z "$depmode" || test -z "$source" || test -z "$object"; then echo "depcomp: Variables source, object and depmode must be set" 1>&2 exit 1 fi # Dependencies for sub/bar.o or sub/bar.obj go into sub/.deps/bar.Po. depfile=${depfile-`echo "$object" | sed 's|[^\\/]*$|'${DEPDIR-.deps}'/&|;s|\.\([^.]*\)$|.P\1|;s|Pobj$|Po|'`} tmpdepfile=${tmpdepfile-`echo "$depfile" | sed 's/\.\([^.]*\)$/.T\1/'`} rm -f "$tmpdepfile" # Avoid interferences from the environment. gccflag= dashmflag= # Some modes work just like other modes, but use different flags. We # parameterize here, but still list the modes in the big case below, # to make depend.m4 easier to write. Note that we *cannot* use a case # here, because this file can only contain one case statement. if test "$depmode" = hp; then # HP compiler uses -M and no extra arg. gccflag=-M depmode=gcc fi if test "$depmode" = dashXmstdout; then # This is just like dashmstdout with a different argument. dashmflag=-xM depmode=dashmstdout fi cygpath_u="cygpath -u -f -" if test "$depmode" = msvcmsys; then # This is just like msvisualcpp but w/o cygpath translation. # Just convert the backslash-escaped backslashes to single forward # slashes to satisfy depend.m4 cygpath_u='sed s,\\\\,/,g' depmode=msvisualcpp fi if test "$depmode" = msvc7msys; then # This is just like msvc7 but w/o cygpath translation. # Just convert the backslash-escaped backslashes to single forward # slashes to satisfy depend.m4 cygpath_u='sed s,\\\\,/,g' depmode=msvc7 fi if test "$depmode" = xlc; then # IBM C/C++ Compilers xlc/xlC can output gcc-like dependency information. gccflag=-qmakedep=gcc,-MF depmode=gcc fi case "$depmode" in gcc3) ## gcc 3 implements dependency tracking that does exactly what ## we want. Yay! Note: for some reason libtool 1.4 doesn't like ## it if -MD -MP comes after the -MF stuff. Hmm. ## Unfortunately, FreeBSD c89 acceptance of flags depends upon ## the command line argument order; so add the flags where they ## appear in depend2.am. Note that the slowdown incurred here ## affects only configure: in makefiles, %FASTDEP% shortcuts this. for arg do case $arg in -c) set fnord "$@" -MT "$object" -MD -MP -MF "$tmpdepfile" "$arg" ;; *) set fnord "$@" "$arg" ;; esac shift # fnord shift # $arg done "$@" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi mv "$tmpdepfile" "$depfile" ;; gcc) ## Note that this doesn't just cater to obsosete pre-3.x GCC compilers. ## but also to in-use compilers like IMB xlc/xlC and the HP C compiler. ## (see the conditional assignment to $gccflag above). ## There are various ways to get dependency output from gcc. Here's ## why we pick this rather obscure method: ## - Don't want to use -MD because we'd like the dependencies to end ## up in a subdir. Having to rename by hand is ugly. ## (We might end up doing this anyway to support other compilers.) ## - The DEPENDENCIES_OUTPUT environment variable makes gcc act like ## -MM, not -M (despite what the docs say). Also, it might not be ## supported by the other compilers which use the 'gcc' depmode. ## - Using -M directly means running the compiler twice (even worse ## than renaming). if test -z "$gccflag"; then gccflag=-MD, fi "$@" -Wp,"$gccflag$tmpdepfile" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" echo "$object : \\" > "$depfile" # The second -e expression handles DOS-style file names with drive # letters. sed -e 's/^[^:]*: / /' \ -e 's/^['$alpha']:\/[^:]*: / /' < "$tmpdepfile" >> "$depfile" ## This next piece of magic avoids the "deleted header file" problem. ## The problem is that when a header file which appears in a .P file ## is deleted, the dependency causes make to die (because there is ## typically no way to rebuild the header). We avoid this by adding ## dummy dependencies for each header file. Too bad gcc doesn't do ## this for us directly. ## Some versions of gcc put a space before the ':'. On the theory ## that the space means something, we add a space to the output as ## well. hp depmode also adds that space, but also prefixes the VPATH ## to the object. Take care to not repeat it in the output. ## Some versions of the HPUX 10.20 sed can't process this invocation ## correctly. Breaking it into two sed invocations is a workaround. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^\\$//' -e '/^$/d' -e "s|.*$object$||" -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; hp) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; sgi) if test "$libtool" = yes; then "$@" "-Wp,-MDupdate,$tmpdepfile" else "$@" -MDupdate "$tmpdepfile" fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" if test -f "$tmpdepfile"; then # yes, the sourcefile depend on other files echo "$object : \\" > "$depfile" # Clip off the initial element (the dependent). Don't try to be # clever and replace this with sed code, as IRIX sed won't handle # lines with more than a fixed number of characters (4096 in # IRIX 6.2 sed, 8192 in IRIX 6.5). We also remove comment lines; # the IRIX cc adds comments like '#:fec' to the end of the # dependency line. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' \ | tr "$nl" ' ' >> "$depfile" echo >> "$depfile" # The second pass generates a dummy entry for each header file. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^.*\.o://' -e 's/#.*$//' -e '/^$/ d' -e 's/$/:/' \ >> "$depfile" else make_dummy_depfile fi rm -f "$tmpdepfile" ;; xlc) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; aix) # The C for AIX Compiler uses -M and outputs the dependencies # in a .u file. In older versions, this file always lives in the # current directory. Also, the AIX compiler puts '$object:' at the # start of each line; $object doesn't have directory information. # Version 6 uses the directory in both cases. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then tmpdepfile1=$dir$base.u tmpdepfile2=$base.u tmpdepfile3=$dir.libs/$base.u "$@" -Wc,-M else tmpdepfile1=$dir$base.u tmpdepfile2=$dir$base.u tmpdepfile3=$dir$base.u "$@" -M fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" do test -f "$tmpdepfile" && break done aix_post_process_depfile ;; tcc) # tcc (Tiny C Compiler) understand '-MD -MF file' since version 0.9.26 # FIXME: That version still under development at the moment of writing. # Make that this statement remains true also for stable, released # versions. # It will wrap lines (doesn't matter whether long or short) with a # trailing '\', as in: # # foo.o : \ # foo.c \ # foo.h \ # # It will put a trailing '\' even on the last line, and will use leading # spaces rather than leading tabs (at least since its commit 0394caf7 # "Emit spaces for -MD"). "$@" -MD -MF "$tmpdepfile" stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" # Each non-empty line is of the form 'foo.o : \' or ' dep.h \'. # We have to change lines of the first kind to '$object: \'. sed -e "s|.*:|$object :|" < "$tmpdepfile" > "$depfile" # And for each line of the second kind, we have to emit a 'dep.h:' # dummy dependency, to avoid the deleted-header problem. sed -n -e 's|^ *\(.*\) *\\$|\1:|p' < "$tmpdepfile" >> "$depfile" rm -f "$tmpdepfile" ;; ## The order of this option in the case statement is important, since the ## shell code in configure will try each of these formats in the order ## listed in this file. A plain '-MD' option would be understood by many ## compilers, so we must ensure this comes after the gcc and icc options. pgcc) # Portland's C compiler understands '-MD'. # Will always output deps to 'file.d' where file is the root name of the # source file under compilation, even if file resides in a subdirectory. # The object file name does not affect the name of the '.d' file. # pgcc 10.2 will output # foo.o: sub/foo.c sub/foo.h # and will wrap long lines using '\' : # foo.o: sub/foo.c ... \ # sub/foo.h ... \ # ... set_dir_from "$object" # Use the source, not the object, to determine the base name, since # that's sadly what pgcc will do too. set_base_from "$source" tmpdepfile=$base.d # For projects that build the same source file twice into different object # files, the pgcc approach of using the *source* file root name can cause # problems in parallel builds. Use a locking strategy to avoid stomping on # the same $tmpdepfile. lockdir=$base.d-lock trap " echo '$0: caught signal, cleaning up...' >&2 rmdir '$lockdir' exit 1 " 1 2 13 15 numtries=100 i=$numtries while test $i -gt 0; do # mkdir is a portable test-and-set. if mkdir "$lockdir" 2>/dev/null; then # This process acquired the lock. "$@" -MD stat=$? # Release the lock. rmdir "$lockdir" break else # If the lock is being held by a different process, wait # until the winning process is done or we timeout. while test -d "$lockdir" && test $i -gt 0; do sleep 1 i=`expr $i - 1` done fi i=`expr $i - 1` done trap - 1 2 13 15 if test $i -le 0; then echo "$0: failed to acquire lock after $numtries attempts" >&2 echo "$0: check lockdir '$lockdir'" >&2 exit 1 fi if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" # Each line is of the form `foo.o: dependent.h', # or `foo.o: dep1.h dep2.h \', or ` dep3.h dep4.h \'. # Do two passes, one to just change these to # `$object: dependent.h' and one to simply `dependent.h:'. sed "s,^[^:]*:,$object :," < "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process this invocation # correctly. Breaking it into two sed invocations is a workaround. sed 's,^[^:]*: \(.*\)$,\1,;s/^\\$//;/^$/d;/:$/d' < "$tmpdepfile" \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; hp2) # The "hp" stanza above does not work with aCC (C++) and HP's ia64 # compilers, which have integrated preprocessors. The correct option # to use with these is +Maked; it writes dependencies to a file named # 'foo.d', which lands next to the object file, wherever that # happens to be. # Much of this is similar to the tru64 case; see comments there. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then tmpdepfile1=$dir$base.d tmpdepfile2=$dir.libs/$base.d "$@" -Wc,+Maked else tmpdepfile1=$dir$base.d tmpdepfile2=$dir$base.d "$@" +Maked fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" do test -f "$tmpdepfile" && break done if test -f "$tmpdepfile"; then sed -e "s,^.*\.[$lower]*:,$object:," "$tmpdepfile" > "$depfile" # Add 'dependent.h:' lines. sed -ne '2,${ s/^ *// s/ \\*$// s/$/:/ p }' "$tmpdepfile" >> "$depfile" else make_dummy_depfile fi rm -f "$tmpdepfile" "$tmpdepfile2" ;; tru64) # The Tru64 compiler uses -MD to generate dependencies as a side # effect. 'cc -MD -o foo.o ...' puts the dependencies into 'foo.o.d'. # At least on Alpha/Redhat 6.1, Compaq CCC V6.2-504 seems to put # dependencies in 'foo.d' instead, so we check for that too. # Subdirectories are respected. set_dir_from "$object" set_base_from "$object" if test "$libtool" = yes; then # Libtool generates 2 separate objects for the 2 libraries. These # two compilations output dependencies in $dir.libs/$base.o.d and # in $dir$base.o.d. We have to check for both files, because # one of the two compilations can be disabled. We should prefer # $dir$base.o.d over $dir.libs/$base.o.d because the latter is # automatically cleaned when .libs/ is deleted, while ignoring # the former would cause a distcleancheck panic. tmpdepfile1=$dir$base.o.d # libtool 1.5 tmpdepfile2=$dir.libs/$base.o.d # Likewise. tmpdepfile3=$dir.libs/$base.d # Compaq CCC V6.2-504 "$@" -Wc,-MD else tmpdepfile1=$dir$base.d tmpdepfile2=$dir$base.d tmpdepfile3=$dir$base.d "$@" -MD fi stat=$? if test $stat -ne 0; then rm -f "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" exit $stat fi for tmpdepfile in "$tmpdepfile1" "$tmpdepfile2" "$tmpdepfile3" do test -f "$tmpdepfile" && break done # Same post-processing that is required for AIX mode. aix_post_process_depfile ;; msvc7) if test "$libtool" = yes; then showIncludes=-Wc,-showIncludes else showIncludes=-showIncludes fi "$@" $showIncludes > "$tmpdepfile" stat=$? grep -v '^Note: including file: ' "$tmpdepfile" if test $stat -ne 0; then rm -f "$tmpdepfile" exit $stat fi rm -f "$depfile" echo "$object : \\" > "$depfile" # The first sed program below extracts the file names and escapes # backslashes for cygpath. The second sed program outputs the file # name when reading, but also accumulates all include files in the # hold buffer in order to output them again at the end. This only # works with sed implementations that can handle large buffers. sed < "$tmpdepfile" -n ' /^Note: including file: *\(.*\)/ { s//\1/ s/\\/\\\\/g p }' | $cygpath_u | sort -u | sed -n ' s/ /\\ /g s/\(.*\)/'"$tab"'\1 \\/p s/.\(.*\) \\/\1:/ H $ { s/.*/'"$tab"'/ G p }' >> "$depfile" echo >> "$depfile" # make sure the fragment doesn't end with a backslash rm -f "$tmpdepfile" ;; msvc7msys) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; #nosideeffect) # This comment above is used by automake to tell side-effect # dependency tracking mechanisms from slower ones. dashmstdout) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout, regardless of -o. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # Remove '-o $object'. IFS=" " for arg do case $arg in -o) shift ;; $object) shift ;; *) set fnord "$@" "$arg" shift # fnord shift # $arg ;; esac done test -z "$dashmflag" && dashmflag=-M # Require at least two characters before searching for ':' # in the target name. This is to cope with DOS-style filenames: # a dependency such as 'c:/foo/bar' could be seen as target 'c' otherwise. "$@" $dashmflag | sed "s|^[$tab ]*[^:$tab ][^:][^:]*:[$tab ]*|$object: |" > "$tmpdepfile" rm -f "$depfile" cat < "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process this sed invocation # correctly. Breaking it into two sed invocations is a workaround. tr ' ' "$nl" < "$tmpdepfile" \ | sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; dashXmstdout) # This case only exists to satisfy depend.m4. It is never actually # run, as this mode is specially recognized in the preamble. exit 1 ;; makedepend) "$@" || exit $? # Remove any Libtool call if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # X makedepend shift cleared=no eat=no for arg do case $cleared in no) set ""; shift cleared=yes ;; esac if test $eat = yes; then eat=no continue fi case "$arg" in -D*|-I*) set fnord "$@" "$arg"; shift ;; # Strip any option that makedepend may not understand. Remove # the object too, otherwise makedepend will parse it as a source file. -arch) eat=yes ;; -*|$object) ;; *) set fnord "$@" "$arg"; shift ;; esac done obj_suffix=`echo "$object" | sed 's/^.*\././'` touch "$tmpdepfile" ${MAKEDEPEND-makedepend} -o"$obj_suffix" -f"$tmpdepfile" "$@" rm -f "$depfile" # makedepend may prepend the VPATH from the source file name to the object. # No need to regex-escape $object, excess matching of '.' is harmless. sed "s|^.*\($object *:\)|\1|" "$tmpdepfile" > "$depfile" # Some versions of the HPUX 10.20 sed can't process the last invocation # correctly. Breaking it into two sed invocations is a workaround. sed '1,2d' "$tmpdepfile" \ | tr ' ' "$nl" \ | sed -e 's/^\\$//' -e '/^$/d' -e '/:$/d' \ | sed -e 's/$/ :/' >> "$depfile" rm -f "$tmpdepfile" "$tmpdepfile".bak ;; cpp) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi # Remove '-o $object'. IFS=" " for arg do case $arg in -o) shift ;; $object) shift ;; *) set fnord "$@" "$arg" shift # fnord shift # $arg ;; esac done "$@" -E \ | sed -n -e '/^# [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ -e '/^#line [0-9][0-9]* "\([^"]*\)".*/ s:: \1 \\:p' \ | sed '$ s: \\$::' > "$tmpdepfile" rm -f "$depfile" echo "$object : \\" > "$depfile" cat < "$tmpdepfile" >> "$depfile" sed < "$tmpdepfile" '/^$/d;s/^ //;s/ \\$//;s/$/ :/' >> "$depfile" rm -f "$tmpdepfile" ;; msvisualcpp) # Important note: in order to support this mode, a compiler *must* # always write the preprocessed file to stdout. "$@" || exit $? # Remove the call to Libtool. if test "$libtool" = yes; then while test "X$1" != 'X--mode=compile'; do shift done shift fi IFS=" " for arg do case "$arg" in -o) shift ;; $object) shift ;; "-Gm"|"/Gm"|"-Gi"|"/Gi"|"-ZI"|"/ZI") set fnord "$@" shift shift ;; *) set fnord "$@" "$arg" shift shift ;; esac done "$@" -E 2>/dev/null | sed -n '/^#line [0-9][0-9]* "\([^"]*\)"/ s::\1:p' | $cygpath_u | sort -u > "$tmpdepfile" rm -f "$depfile" echo "$object : \\" > "$depfile" sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::'"$tab"'\1 \\:p' >> "$depfile" echo "$tab" >> "$depfile" sed < "$tmpdepfile" -n -e 's% %\\ %g' -e '/^\(.*\)$/ s::\1\::p' >> "$depfile" rm -f "$tmpdepfile" ;; msvcmsys) # This case exists only to let depend.m4 do its work. It works by # looking at the text of this script. This case will never be run, # since it is checked for above. exit 1 ;; none) exec "$@" ;; *) echo "Unknown depmode $depmode" 1>&2 exit 1 ;; esac exit 0 # Local Variables: # mode: shell-script # sh-indentation: 2 # eval: (add-hook 'before-save-hook 'time-stamp) # time-stamp-start: "scriptversion=" # time-stamp-format: "%:y-%02m-%02d.%02H" # time-stamp-time-zone: "UTC0" # time-stamp-end: "; # UTC" # End: dballe-8.6/src/0000755000175000017500000000000013602152021010372 500000000000000dballe-8.6/src/dbaexport0000755000175000017500000001110513565546566012263 00000000000000#!/usr/bin/python3 import argparse import logging import os import sys import io try: import dballe HAS_DBALLE = True except ImportError: HAS_DBALLE = False log = logging.getLogger("main") class CommandError(Exception): pass def setup_logging(args): FORMAT = "%(levelname)s: %(message)s" if args.verbose: logging.basicConfig(level=logging.INFO, stream=sys.stderr, format=FORMAT) else: logging.basicConfig(level=logging.WARNING, stream=sys.stderr, format=FORMAT) def get_db(args): if not args.url: url = os.environ.get("DBA_DB", "") else: url = args.url if not url: raise CommandError("Cannot find a database to connect to: --url is not specified and $DBA_DB is not set") return dballe.DB.connect(url) def get_query(args): return dict(q.split("=", 1) for q in args.query) def get_outfd(args, text=False): """ from __future__ import unicode_literals Get the output file descriptor """ if args.outfile: if text: return io.open(args.outfile, "wt", encoding="utf-8") else: return io.open(args.outfile, "wb") else: if text: return io.open(sys.stdout.fileno(), "wt", encoding="utf-8", closefd=False) else: return io.open(sys.stdout.fileno(), "wb", closefd=False) if __name__ == "__main__": parser = argparse.ArgumentParser(description='Export data from a DB-All.e database.') parser.add_argument("--verbose", action="store_true", help="verbose output") parser.add_argument("--url", "--dsn", type=str, metavar="url", help="URL-like database definition, to use for connecting to the DB-All.e database" " (can also be specified in the environment as DBA_DB)") parser.add_argument("--user", type=str, metavar="name", help="username to use for connecting to the DB-All.e database") parser.add_argument("--pass", type=str, metavar="password", dest="password", help="password to use for connecting to the DB-All.e database") parser.add_argument("--outfile", "-o", type=str, metavar="file", help="output file. Default is standard output, if supported") subparsers = parser.add_subparsers(dest="format", help="output formats") # Use a byte-string for subparser name to avoid seeing u'csv' in output. # Remove in python3 parser_csv = subparsers.add_parser("csv", help="export data as CSV") parser_csv.add_argument("query", nargs="*", metavar="key=val", help="DB-All.e query to select data to export") parser_gnur = subparsers.add_parser("gnur", help="export data as GNU R workspace") parser_gnur.add_argument("query", nargs="*", metavar="key=val", help="DB-All.e query to select data to export") parser_bufr = subparsers.add_parser("bufr", help="export data as BUFR") parser_bufr.add_argument("query", nargs="*", metavar="key=val", help="DB-All.e query to select data to export") parser_bufr.add_argument("--generic", action="store_true", help="export generic BUFR messages") parser_crex = subparsers.add_parser("crex", help="export data as CREX") parser_crex.add_argument("query", nargs="*", metavar="key=val", help="DB-All.e query to select data to export") parser_crex.add_argument("--generic", action="store_true", help="export generic BUFR messages") args = parser.parse_args() try: setup_logging(args) if not HAS_DBALLE: raise CommandError("This command requires the dballe python module to run") db = get_db(args) query = get_query(args) if args.format == "csv": import dballe.dbacsv with get_outfd(args, text=True) as fd: dballe.dbacsv.export(db, query, fd) elif args.format == "bufr": if not args.outfile: raise CommandError("BUFR output requires --outfile") db.export_to_file(query, "BUFR", args.outfile, generic=args.generic) elif args.format == "crex": if not args.outfile: raise CommandError("CREX output requires --outfile") db.export_to_file(query, "CREX", args.outfile, generic=args.generic) else: raise CommandError("Exporter for {} is not available.".format(args.format)) except CommandError as e: log.error("%s", e) sys.exit(1) dballe-8.6/src/Makefile.am0000644000175000017500000000123613554564112012366 00000000000000## Process this file with automake to produce Makefile.in DBALLELIBS = ../dballe/libdballe.la AM_CPPFLAGS = -I$(top_srcdir) $(WREPORT_CFLAGS) -Werror if FILE_OFFSET_BITS_64 AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64 endif bin_PROGRAMS = dbatbl dbamsg dist_bin_SCRIPTS = dbaexport dbatbl_SOURCES = dbatbl.cc dbatbl_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbatbl_DEPENDENCIES = $(DBALLELIBS) dbamsg_SOURCES = dbamsg.cc dbamsg_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbamsg_DEPENDENCIES = $(DBALLELIBS) bin_PROGRAMS += dbadb dbadb_SOURCES = dbadb.cc dbadb_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbadb_DEPENDENCIES = $(DBALLELIBS) dballe-8.6/src/dbadb.cc0000644000175000017500000003557213574460031011704 00000000000000#include #include #include #include #include #include #include #include #include #include #include using namespace dballe; using namespace dballe::cmdline; using namespace wreport; using namespace std; // Command line parser variables struct cmdline::ReaderOptions readeropts; static const char* op_output_template = ""; const char* op_output_type = "bufr"; const char* op_report = ""; const char* op_url = ""; const char* op_user = ""; const char* op_pass = ""; const char* op_varlist = ""; int op_wipe_first = 0; int op_dump = 0; int op_overwrite = 0; int op_fast = 0; int op_no_attrs = 0; int op_full_pseudoana = 0; int op_verbose = 0; int op_precise_import = 0; int op_wipe_disappear = 0; struct poptOption grepTable[] = { { "category", 0, POPT_ARG_INT, &readeropts.category, 0, "match messages with the given data category", "num" }, { "subcategory", 0, POPT_ARG_INT, &readeropts.subcategory, 0, "match BUFR messages with the given data subcategory", "num" }, { "check-digit", 0, POPT_ARG_INT, &readeropts.checkdigit, 0, "match CREX messages with check digit (if 1) or without check digit (if 0)", "num" }, { "parsable", 0, 0, &readeropts.parsable, 0, "match only messages that can be parsed", 0 }, { "index", 0, POPT_ARG_STRING, &readeropts.index_filter, 0, "match messages with the index in the given range (ex.: 1-5,9,22-30)", "expr" }, POPT_TABLEEND }; struct poptOption dbTable[] = { { "dsn", 0, POPT_ARG_STRING, &op_url, 0, "alias of --url, used for historical compatibility", "url" }, { "url", 0, POPT_ARG_STRING, &op_url, 0, "DSN, or URL-like database definition, to use for connecting to the DB-All.e database (can also be specified in the environment as DBA_DB)", "url" }, { "wipe-first", 0, POPT_ARG_NONE, &op_wipe_first, 0, "wipe database before any other action" }, POPT_TABLEEND }; static std::shared_ptr connect() { const char* chosen_url; /* If url is missing, look in the environment */ if (op_url[0] == 0) { chosen_url = getenv("DBA_DB"); if (chosen_url == NULL) throw error_consistency("no database specified"); } else chosen_url = op_url; /* If url looks like a url, treat it accordingly */ auto options = DBConnectOptions::create(chosen_url); auto db = dynamic_pointer_cast(DB::connect(*options)); // Wipe database if requested if (op_wipe_first) db->reset(); return db; } // Command line parsing wrappers for Dbadb methods struct DatabaseCmd : public cmdline::Subcommand { void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &dbTable, 0, "Options used to connect to the database", 0 }); } }; struct DumpCmd : public DatabaseCmd { DumpCmd() { names.push_back("dump"); usage = "dump [options] [queryparm1=val1 [queryparm2=val2 [...]]]"; desc = "Dump data from the database"; longdesc = "Query parameters are the same of the Fortran API. " "Please see the section \"Input and output parameters -- For data " "related action routines\" of the Fortran API documentation for a " "complete list."; } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); /* Create the query */ core::Query query; dba_cmdline_get_query(optCon, query); auto db = connect(); Dbadb dbadb(*db); return dbadb.do_dump(query, stdout); } }; struct StationsCmd : public DatabaseCmd { StationsCmd() { names.push_back("stations"); usage = "stations [options] [queryparm1=val1 [queryparm2=val2 [...]]]"; desc = "List the stations present in the database"; longdesc = "Query parameters are the same of the Fortran API. " "Please see the section \"Input and output parameters -- For data " "related action routines\" of the Fortran API documentation for a " "complete list."; } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); /* Create the query */ core::Query query; dba_cmdline_get_query(optCon, query); auto db = connect(); Dbadb dbadb(*db); return dbadb.do_stations(query, stdout); } }; /// Create / empty the database struct WipeCmd : public DatabaseCmd { WipeCmd() { names.push_back("wipe"); usage = "wipe [options] [optional rep_memo description file]"; desc = "Reinitialise the database, removing all data"; longdesc = "Reinitialisation is done using the given report code description file. " "If no file is provided, a default version is used"; } void add_to_optable(std::vector& opts) const override { DatabaseCmd::add_to_optable(opts); opts.push_back({ "disappear", 0, POPT_ARG_NONE, &op_wipe_disappear, 0, "just remove the DB-All.e data and tables", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); /* Get the optional name of the repinfo file */ const char* fname = poptGetArg(optCon); { // Connect first using the current format, and remove all tables. auto db = connect(); db->disappear(); } if (!op_wipe_disappear) { // Recreate tables auto db = connect(); db->reset(fname); } return 0; } }; /// Perform database cleanup maintenance struct CleanupCmd : public DatabaseCmd { CleanupCmd() { names.push_back("cleanup"); usage = "cleanup [options]"; desc = "Perform database cleanup operations"; longdesc = "The only operation currently performed by this command is " "deleting stations that have no values. If more will be added in " "the future, they will be documented here."; } int main(poptContext optCon) { auto db = connect(); db->vacuum(); return 0; } }; /// Update repinfo information in the database struct RepinfoCmd : public DatabaseCmd { RepinfoCmd() { names.push_back("repinfo"); usage = "repinfo [options] [filename]"; desc = "Update the report information table"; longdesc = "Update the report information table with the data from the given " "report code description file. " "If no file is provided, a default version is used"; } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); auto db = connect(); /* Get the optional name of the repinfo file. If missing, the default will be used */ const char* fname = poptGetArg(optCon); auto tr = dynamic_pointer_cast(db->transaction()); int added, deleted, updated; tr->update_repinfo(fname, &added, &deleted, &updated); tr->commit(); printf("Update completed: %d added, %d deleted, %d updated.\n", added, deleted, updated); return 0; } }; struct ImportCmd : public DatabaseCmd { ImportCmd() { names.push_back("import"); usage = "import [options] [filter] filename [filename [ ... ] ]"; desc = "Import data into the database"; } void add_to_optable(std::vector& opts) const override { DatabaseCmd::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'csv', 'json')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ "overwrite", 'f', POPT_ARG_NONE, &op_overwrite, 0, "overwrite existing data", 0 }); opts.push_back({ "report", 'r', POPT_ARG_STRING, &op_report, 0, "force data to be of this type of report", "rep" }); opts.push_back({ "fast", 0, POPT_ARG_NONE, &op_fast, 0, "Prefer speed to transactional integrity: if the import is interrupted," " the database needs to be wiped and recreated.", 0 }); opts.push_back({ "no-attrs", 0, POPT_ARG_NONE, &op_no_attrs, 0, "do not import data attributes", 0 }); opts.push_back({ "full-pseudoana", 0, POPT_ARG_NONE, &op_full_pseudoana, 0, "merge pseudoana extra values with the ones already existing in the database", 0 }); opts.push_back({ "precise", 0, 0, &op_precise_import, 0, "import messages using precise contexts instead of standard ones", 0 }); opts.push_back({ "varlist", 0, POPT_ARG_STRING, &op_varlist, 0, "only import variables with the given varcode(s)", "varlist" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { // Throw away the command name poptGetArg(optCon); cmdline::Reader reader(readeropts); reader.verbose = op_verbose; // Configure the reader core::Query query; if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); reader.import_opts.simplified = !op_precise_import; // Configure the importer auto opts = DBImportOptions::create(); if (op_overwrite) opts->overwrite = true; if (op_fast) setenv("DBA_INSECURE_SQLITE", "true", true); if (!op_no_attrs) opts->import_attributes = true; if (op_full_pseudoana) opts->update_station = true; if (op_varlist[0]) resolve_varlist(op_varlist, [&](wreport::Varcode code) { opts->varlist.push_back(code); }); auto db = connect(); if (strcmp(op_report, "") != 0) opts->report = op_report; Dbadb dbadb(*db); return dbadb.do_import(get_filenames(optCon), reader, *opts); } }; struct ExportCmd : public DatabaseCmd { ExportCmd() { names.push_back("export"); usage = "export [options] [queryparm1=val1 [queryparm2=val2 [...]]]"; desc = "Export data from the database"; longdesc = "Query parameters are the same of the Fortran API. " "Please see the section \"Input and output parameters -- For data " "related action routines\" of the Fortran API documentation for a " "complete list.\n\n" "The database is specified with --url or with the DBA_DB" " environment variable"; } void add_to_optable(std::vector& opts) const override { DatabaseCmd::add_to_optable(opts); opts.push_back({ "report", 'r', POPT_ARG_STRING, &op_report, 0, "force exported data to be of this type of report", "rep" }); opts.push_back({ "dest", 'd', POPT_ARG_STRING, &op_output_type, 0, "format of the data in output ('bufr', 'crex')", "type" }); opts.push_back({ "template", 't', POPT_ARG_STRING, &op_output_template, 0, "template of the data in output (autoselect if not specified, 'list' gives a list)", "name" }); opts.push_back({ "dump", 0, POPT_ARG_NONE, &op_dump, 0, "dump data to be encoded instead of encoding it", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); if (strcmp(op_output_template, "list") == 0) { list_templates(); return 0; } // Reat the query from command line core::Query query; dba_cmdline_get_query(optCon, query); auto db = connect(); Dbadb dbadb(*db); const char* forced_repmemo = NULL; if (strcmp(op_report, "") != 0) forced_repmemo = op_report; if (op_dump) { return dbadb.do_export_dump(query, stdout); } else { Encoding type = File::parse_encoding(op_output_type); auto file = File::create(type, stdout, false, "w"); return dbadb.do_export(query, *file, op_output_template, forced_repmemo); } } }; struct DeleteCmd : public DatabaseCmd { DeleteCmd() { names.push_back("delete"); usage = "delete [options] [queryparm1=val1 [queryparm2=val2 [...]]]"; desc = "Delete all the data matching the given query parameters"; longdesc = "Query parameters are the same of the Fortran API. " "Please see the section \"Input and output parameters -- For data " "related action routines\" of the Fortran API documentation for a " "complete list."; } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); if (poptPeekArg(optCon) == NULL) dba_cmdline_error(optCon, "you need to specify some query parameters"); /* Add the query data from commandline */ core::Query query; dba_cmdline_get_query(optCon, query); auto db = connect(); // TODO: check that there is something db->remove_data(query); return 0; } }; struct InfoCmd : public DatabaseCmd { InfoCmd() { names.push_back("info"); usage = "info"; desc = "Print information about the database"; } int main(poptContext optCon) override { // Throw away the command name poptGetArg(optCon); auto db = connect(); string default_format = db::format_format(db::DB::get_default_format()); fprintf(stdout, "Default format for new DBs: %s\n", default_format.c_str()); db->print_info(stdout); return 0; } }; int main (int argc, const char* argv[]) { Command dbadb; dbadb.name = "dbadb"; dbadb.desc = "Manage the DB-ALLe database"; dbadb.longdesc = "It allows to initialise the database, dump its contents and import and export data " "using BUFR, or CREX encoding"; dbadb.add_subcommand(new DumpCmd); dbadb.add_subcommand(new StationsCmd); dbadb.add_subcommand(new WipeCmd); dbadb.add_subcommand(new CleanupCmd); dbadb.add_subcommand(new RepinfoCmd); dbadb.add_subcommand(new ImportCmd); dbadb.add_subcommand(new ExportCmd); dbadb.add_subcommand(new DeleteCmd); dbadb.add_subcommand(new InfoCmd); return dbadb.main(argc, argv); } /* vim:set ts=4 sw=4: */ dballe-8.6/src/Makefile.in0000644000175000017500000006323113602151750012374 00000000000000# Makefile.in generated by automake 1.16.1 from Makefile.am. # @configure_input@ # Copyright (C) 1994-2018 Free Software Foundation, Inc. # This Makefile.in is free software; the Free Software Foundation # gives unlimited permission to copy and/or distribute it, # with or without modifications, as long as this notice is preserved. # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY, to the extent permitted by law; without # even the implied warranty of MERCHANTABILITY or FITNESS FOR A # PARTICULAR PURPOSE. @SET_MAKE@ VPATH = @srcdir@ am__is_gnu_make = { \ if test -z '$(MAKELEVEL)'; then \ false; \ elif test -n '$(MAKE_HOST)'; then \ true; \ elif test -n '$(MAKE_VERSION)' && test -n '$(CURDIR)'; then \ true; \ else \ false; \ fi; \ } am__make_running_with_option = \ case $${target_option-} in \ ?) ;; \ *) echo "am__make_running_with_option: internal error: invalid" \ "target option '$${target_option-}' specified" >&2; \ exit 1;; \ esac; \ has_opt=no; \ sane_makeflags=$$MAKEFLAGS; \ if $(am__is_gnu_make); then \ sane_makeflags=$$MFLAGS; \ else \ case $$MAKEFLAGS in \ *\\[\ \ ]*) \ bs=\\; \ sane_makeflags=`printf '%s\n' "$$MAKEFLAGS" \ | sed "s/$$bs$$bs[$$bs $$bs ]*//g"`;; \ esac; \ fi; \ skip_next=no; \ strip_trailopt () \ { \ flg=`printf '%s\n' "$$flg" | sed "s/$$1.*$$//"`; \ }; \ for flg in $$sane_makeflags; do \ test $$skip_next = yes && { skip_next=no; continue; }; \ case $$flg in \ *=*|--*) continue;; \ -*I) strip_trailopt 'I'; skip_next=yes;; \ -*I?*) strip_trailopt 'I';; \ -*O) strip_trailopt 'O'; skip_next=yes;; \ -*O?*) strip_trailopt 'O';; \ -*l) strip_trailopt 'l'; skip_next=yes;; \ -*l?*) strip_trailopt 'l';; \ -[dEDm]) skip_next=yes;; \ -[JT]) skip_next=yes;; \ esac; \ case $$flg in \ *$$target_option*) has_opt=yes; break;; \ esac; \ done; \ test $$has_opt = yes am__make_dryrun = (target_option=n; $(am__make_running_with_option)) am__make_keepgoing = (target_option=k; $(am__make_running_with_option)) pkgdatadir = $(datadir)/@PACKAGE@ pkgincludedir = $(includedir)/@PACKAGE@ pkglibdir = $(libdir)/@PACKAGE@ pkglibexecdir = $(libexecdir)/@PACKAGE@ am__cd = CDPATH="$${ZSH_VERSION+.}$(PATH_SEPARATOR)" && cd install_sh_DATA = $(install_sh) -c -m 644 install_sh_PROGRAM = $(install_sh) -c install_sh_SCRIPT = $(install_sh) -c INSTALL_HEADER = $(INSTALL_DATA) transform = $(program_transform_name) NORMAL_INSTALL = : PRE_INSTALL = : POST_INSTALL = : NORMAL_UNINSTALL = : PRE_UNINSTALL = : POST_UNINSTALL = : build_triplet = @build@ host_triplet = @host@ @FILE_OFFSET_BITS_64_TRUE@am__append_1 = -D_FILE_OFFSET_BITS=64 bin_PROGRAMS = dbatbl$(EXEEXT) dbamsg$(EXEEXT) dbadb$(EXEEXT) subdir = src ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 am__aclocal_m4_deps = $(top_srcdir)/m4/ax_append_flag.m4 \ $(top_srcdir)/m4/ax_cflags_warn_all.m4 \ $(top_srcdir)/m4/ax_python_module.m4 \ $(top_srcdir)/m4/libtool.m4 $(top_srcdir)/m4/ltoptions.m4 \ $(top_srcdir)/m4/ltsugar.m4 $(top_srcdir)/m4/ltversion.m4 \ $(top_srcdir)/m4/lt~obsolete.m4 \ $(top_srcdir)/m4/m4_ax_cxx_compile_stdcxx_11.m4 \ $(top_srcdir)/m4/pkg.m4 $(top_srcdir)/m4/python.m4 \ $(top_srcdir)/configure.ac am__configure_deps = $(am__aclocal_m4_deps) $(CONFIGURE_DEPENDENCIES) \ $(ACLOCAL_M4) DIST_COMMON = $(srcdir)/Makefile.am $(dist_bin_SCRIPTS) \ $(am__DIST_COMMON) mkinstalldirs = $(install_sh) -d CONFIG_HEADER = $(top_builddir)/config.h CONFIG_CLEAN_FILES = CONFIG_CLEAN_VPATH_FILES = am__installdirs = "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)" PROGRAMS = $(bin_PROGRAMS) am_dbadb_OBJECTS = dbadb.$(OBJEXT) dbadb_OBJECTS = $(am_dbadb_OBJECTS) dbadb_LDADD = $(LDADD) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent am__v_lt_1 = dbadb_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(dbadb_LDFLAGS) $(LDFLAGS) -o $@ am_dbamsg_OBJECTS = dbamsg.$(OBJEXT) dbamsg_OBJECTS = $(am_dbamsg_OBJECTS) dbamsg_LDADD = $(LDADD) dbamsg_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(dbamsg_LDFLAGS) $(LDFLAGS) -o $@ am_dbatbl_OBJECTS = dbatbl.$(OBJEXT) dbatbl_OBJECTS = $(am_dbatbl_OBJECTS) dbatbl_LDADD = $(LDADD) dbatbl_LINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(dbatbl_LDFLAGS) $(LDFLAGS) -o $@ am__vpath_adj_setup = srcdirstrip=`echo "$(srcdir)" | sed 's|.|.|g'`; am__vpath_adj = case $$p in \ $(srcdir)/*) f=`echo "$$p" | sed "s|^$$srcdirstrip/||"`;; \ *) f=$$p;; \ esac; am__strip_dir = f=`echo $$p | sed -e 's|^.*/||'`; am__install_max = 40 am__nobase_strip_setup = \ srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*|]/\\\\&/g'` am__nobase_strip = \ for p in $$list; do echo "$$p"; done | sed -e "s|$$srcdirstrip/||" am__nobase_list = $(am__nobase_strip_setup); \ for p in $$list; do echo "$$p $$p"; done | \ sed "s| $$srcdirstrip/| |;"' / .*\//!s/ .*/ ./; s,\( .*\)/[^/]*$$,\1,' | \ $(AWK) 'BEGIN { files["."] = "" } { files[$$2] = files[$$2] " " $$1; \ if (++n[$$2] == $(am__install_max)) \ { print $$2, files[$$2]; n[$$2] = 0; files[$$2] = "" } } \ END { for (dir in files) print dir, files[dir] }' am__base_list = \ sed '$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;$$!N;s/\n/ /g' | \ sed '$$!N;$$!N;$$!N;$$!N;s/\n/ /g' am__uninstall_files_from_dir = { \ test -z "$$files" \ || { test ! -d "$$dir" && test ! -f "$$dir" && test ! -r "$$dir"; } \ || { echo " ( cd '$$dir' && rm -f" $$files ")"; \ $(am__cd) "$$dir" && rm -f $$files; }; \ } SCRIPTS = $(dist_bin_SCRIPTS) AM_V_P = $(am__v_P_@AM_V@) am__v_P_ = $(am__v_P_@AM_DEFAULT_V@) am__v_P_0 = false am__v_P_1 = : AM_V_GEN = $(am__v_GEN_@AM_V@) am__v_GEN_ = $(am__v_GEN_@AM_DEFAULT_V@) am__v_GEN_0 = @echo " GEN " $@; am__v_GEN_1 = AM_V_at = $(am__v_at_@AM_V@) am__v_at_ = $(am__v_at_@AM_DEFAULT_V@) am__v_at_0 = @ am__v_at_1 = DEFAULT_INCLUDES = -I.@am__isrc@ -I$(top_builddir) depcomp = $(SHELL) $(top_srcdir)/depcomp am__maybe_remake_depfiles = depfiles am__depfiles_remade = ./$(DEPDIR)/dbadb.Po ./$(DEPDIR)/dbamsg.Po \ ./$(DEPDIR)/dbatbl.Po am__mv = mv -f CXXCOMPILE = $(CXX) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(AM_CPPFLAGS) $(CPPFLAGS) $(AM_CXXFLAGS) $(CXXFLAGS) LTCXXCOMPILE = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=compile $(CXX) $(DEFS) \ $(DEFAULT_INCLUDES) $(INCLUDES) $(AM_CPPFLAGS) $(CPPFLAGS) \ $(AM_CXXFLAGS) $(CXXFLAGS) AM_V_CXX = $(am__v_CXX_@AM_V@) am__v_CXX_ = $(am__v_CXX_@AM_DEFAULT_V@) am__v_CXX_0 = @echo " CXX " $@; am__v_CXX_1 = CXXLD = $(CXX) CXXLINK = $(LIBTOOL) $(AM_V_lt) --tag=CXX $(AM_LIBTOOLFLAGS) \ $(LIBTOOLFLAGS) --mode=link $(CXXLD) $(AM_CXXFLAGS) \ $(CXXFLAGS) $(AM_LDFLAGS) $(LDFLAGS) -o $@ AM_V_CXXLD = $(am__v_CXXLD_@AM_V@) am__v_CXXLD_ = $(am__v_CXXLD_@AM_DEFAULT_V@) am__v_CXXLD_0 = @echo " CXXLD " $@; am__v_CXXLD_1 = SOURCES = $(dbadb_SOURCES) $(dbamsg_SOURCES) $(dbatbl_SOURCES) DIST_SOURCES = $(dbadb_SOURCES) $(dbamsg_SOURCES) $(dbatbl_SOURCES) am__can_run_installinfo = \ case $$AM_UPDATE_INFO_DIR in \ n|no|NO) false;; \ *) (install-info --version) >/dev/null 2>&1;; \ esac am__tagged_files = $(HEADERS) $(SOURCES) $(TAGS_FILES) $(LISP) # Read a list of newline-separated strings from the standard input, # and print each of them once, without duplicates. Input order is # *not* preserved. am__uniquify_input = $(AWK) '\ BEGIN { nonempty = 0; } \ { items[$$0] = 1; nonempty = 1; } \ END { if (nonempty) { for (i in items) print i; }; } \ ' # Make sure the list of sources is unique. This is necessary because, # e.g., the same source file might be shared among _SOURCES variables # for different programs/libraries. am__define_uniq_tagged_files = \ list='$(am__tagged_files)'; \ unique=`for i in $$list; do \ if test -f "$$i"; then echo $$i; else echo $(srcdir)/$$i; fi; \ done | $(am__uniquify_input)` ETAGS = etags CTAGS = ctags am__DIST_COMMON = $(srcdir)/Makefile.in $(top_srcdir)/depcomp DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST) ACLOCAL = @ACLOCAL@ AMTAR = @AMTAR@ AM_DEFAULT_VERBOSITY = @AM_DEFAULT_VERBOSITY@ AR = @AR@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ CC = @CC@ CCDEPMODE = @CCDEPMODE@ CFLAGS = @CFLAGS@ CPP = @CPP@ CPPFLAGS = @CPPFLAGS@ CXX = @CXX@ CXXCPP = @CXXCPP@ CXXDEPMODE = @CXXDEPMODE@ CXXFLAGS = @CXXFLAGS@ CYGPATH_W = @CYGPATH_W@ DEFS = @DEFS@ DEPDIR = @DEPDIR@ DLLTOOL = @DLLTOOL@ DOXYGEN = @DOXYGEN@ DSYMUTIL = @DSYMUTIL@ DUMPBIN = @DUMPBIN@ ECHO_C = @ECHO_C@ ECHO_N = @ECHO_N@ ECHO_T = @ECHO_T@ EGREP = @EGREP@ EXEEXT = @EXEEXT@ FC = @FC@ FCFLAGS = @FCFLAGS@ FGREP = @FGREP@ GREP = @GREP@ HAVE_CXX11 = @HAVE_CXX11@ INSTALL = @INSTALL@ INSTALL_DATA = @INSTALL_DATA@ INSTALL_PROGRAM = @INSTALL_PROGRAM@ INSTALL_SCRIPT = @INSTALL_SCRIPT@ INSTALL_STRIP_PROGRAM = @INSTALL_STRIP_PROGRAM@ LD = @LD@ LDFLAGS = @LDFLAGS@ LIBDBALLEF_VERSION_INFO = @LIBDBALLEF_VERSION_INFO@ LIBDBALLE_VERSION_INFO = @LIBDBALLE_VERSION_INFO@ LIBOBJS = @LIBOBJS@ LIBPQ_CFLAGS = @LIBPQ_CFLAGS@ LIBPQ_LIBS = @LIBPQ_LIBS@ LIBS = @LIBS@ LIBTOOL = @LIBTOOL@ LIPO = @LIPO@ LN_S = @LN_S@ LTLIBOBJS = @LTLIBOBJS@ LT_SYS_LIBRARY_PATH = @LT_SYS_LIBRARY_PATH@ MAKEINFO = @MAKEINFO@ MANIFEST_TOOL = @MANIFEST_TOOL@ MKDIR_P = @MKDIR_P@ MYSQL_CFLAGS = @MYSQL_CFLAGS@ MYSQL_LIBS = @MYSQL_LIBS@ NM = @NM@ NMEDIT = @NMEDIT@ OBJDUMP = @OBJDUMP@ OBJEXT = @OBJEXT@ OTOOL = @OTOOL@ OTOOL64 = @OTOOL64@ PACKAGE = @PACKAGE@ PACKAGE_BUGREPORT = @PACKAGE_BUGREPORT@ PACKAGE_NAME = @PACKAGE_NAME@ PACKAGE_STRING = @PACKAGE_STRING@ PACKAGE_TARNAME = @PACKAGE_TARNAME@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ PKGCONFIG_CFLAGS = @PKGCONFIG_CFLAGS@ PKGCONFIG_LIBS = @PKGCONFIG_LIBS@ PKGCONFIG_REQUIRES = @PKGCONFIG_REQUIRES@ PKG_CONFIG = @PKG_CONFIG@ POPT_CFLAGS = @POPT_CFLAGS@ POPT_LIBS = @POPT_LIBS@ PYTHON = @PYTHON@ PYTHON_CFLAGS = @PYTHON_CFLAGS@ PYTHON_EXEC_PREFIX = @PYTHON_EXEC_PREFIX@ PYTHON_MAJOR_VERSION = @PYTHON_MAJOR_VERSION@ PYTHON_PLATFORM = @PYTHON_PLATFORM@ PYTHON_PREFIX = @PYTHON_PREFIX@ PYTHON_VERSION = @PYTHON_VERSION@ RANLIB = @RANLIB@ SED = @SED@ SET_MAKE = @SET_MAKE@ SHELL = @SHELL@ SPHINX_BUILD = @SPHINX_BUILD@ SQLITE3_CFLAGS = @SQLITE3_CFLAGS@ SQLITE3_LIBS = @SQLITE3_LIBS@ STRIP = @STRIP@ VERSION = @VERSION@ WREPORT_CFLAGS = @WREPORT_CFLAGS@ WREPORT_DOXYGEN_DIR = @WREPORT_DOXYGEN_DIR@ WREPORT_DOXYGEN_TAGFILE = @WREPORT_DOXYGEN_TAGFILE@ WREPORT_LIBS = @WREPORT_LIBS@ abs_builddir = @abs_builddir@ abs_srcdir = @abs_srcdir@ abs_top_builddir = @abs_top_builddir@ abs_top_srcdir = @abs_top_srcdir@ ac_ct_AR = @ac_ct_AR@ ac_ct_CC = @ac_ct_CC@ ac_ct_CXX = @ac_ct_CXX@ ac_ct_DUMPBIN = @ac_ct_DUMPBIN@ ac_ct_FC = @ac_ct_FC@ am__include = @am__include@ am__leading_dot = @am__leading_dot@ am__quote = @am__quote@ am__tar = @am__tar@ am__untar = @am__untar@ bindir = @bindir@ build = @build@ build_alias = @build_alias@ build_cpu = @build_cpu@ build_os = @build_os@ build_vendor = @build_vendor@ builddir = @builddir@ confdir = @confdir@ datadir = @datadir@ datarootdir = @datarootdir@ docdir = @docdir@ dvidir = @dvidir@ exec_prefix = @exec_prefix@ have_doxygen = @have_doxygen@ have_gperf = @have_gperf@ host = @host@ host_alias = @host_alias@ host_cpu = @host_cpu@ host_os = @host_os@ host_vendor = @host_vendor@ htmldir = @htmldir@ includedir = @includedir@ infodir = @infodir@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ mysql_config = @mysql_config@ oldincludedir = @oldincludedir@ pdfdir = @pdfdir@ pkgpyexecdir = @pkgpyexecdir@ pkgpythondir = @pkgpythondir@ prefix = @prefix@ program_transform_name = @program_transform_name@ psdir = @psdir@ pyexecdir = @pyexecdir@ pythondir = @pythondir@ runstatedir = @runstatedir@ sbindir = @sbindir@ sharedstatedir = @sharedstatedir@ srcdir = @srcdir@ sysconfdir = @sysconfdir@ tabledir = @tabledir@ target_alias = @target_alias@ top_build_prefix = @top_build_prefix@ top_builddir = @top_builddir@ top_srcdir = @top_srcdir@ DBALLELIBS = ../dballe/libdballe.la AM_CPPFLAGS = -I$(top_srcdir) $(WREPORT_CFLAGS) -Werror \ $(am__append_1) dist_bin_SCRIPTS = dbaexport dbatbl_SOURCES = dbatbl.cc dbatbl_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbatbl_DEPENDENCIES = $(DBALLELIBS) dbamsg_SOURCES = dbamsg.cc dbamsg_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbamsg_DEPENDENCIES = $(DBALLELIBS) dbadb_SOURCES = dbadb.cc dbadb_LDFLAGS = $(DBALLELIBS) $(WREPORT_LIBS) $(POPT_LIBS) dbadb_DEPENDENCIES = $(DBALLELIBS) all: all-am .SUFFIXES: .SUFFIXES: .cc .lo .o .obj $(srcdir)/Makefile.in: $(srcdir)/Makefile.am $(am__configure_deps) @for dep in $?; do \ case '$(am__configure_deps)' in \ *$$dep*) \ ( cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh ) \ && { if test -f $@; then exit 0; else break; fi; }; \ exit 1;; \ esac; \ done; \ echo ' cd $(top_srcdir) && $(AUTOMAKE) --foreign src/Makefile'; \ $(am__cd) $(top_srcdir) && \ $(AUTOMAKE) --foreign src/Makefile Makefile: $(srcdir)/Makefile.in $(top_builddir)/config.status @case '$?' in \ *config.status*) \ cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh;; \ *) \ echo ' cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles)'; \ cd $(top_builddir) && $(SHELL) ./config.status $(subdir)/$@ $(am__maybe_remake_depfiles);; \ esac; $(top_builddir)/config.status: $(top_srcdir)/configure $(CONFIG_STATUS_DEPENDENCIES) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(top_srcdir)/configure: $(am__configure_deps) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(ACLOCAL_M4): $(am__aclocal_m4_deps) cd $(top_builddir) && $(MAKE) $(AM_MAKEFLAGS) am--refresh $(am__aclocal_m4_deps): install-binPROGRAMS: $(bin_PROGRAMS) @$(NORMAL_INSTALL) @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ if test -n "$$list"; then \ echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ fi; \ for p in $$list; do echo "$$p $$p"; done | \ sed 's/$(EXEEXT)$$//' | \ while read p p1; do if test -f $$p \ || test -f $$p1 \ ; then echo "$$p"; echo "$$p"; else :; fi; \ done | \ sed -e 'p;s,.*/,,;n;h' \ -e 's|.*|.|' \ -e 'p;x;s,.*/,,;s/$(EXEEXT)$$//;$(transform);s/$$/$(EXEEXT)/' | \ sed 'N;N;N;s,\n, ,g' | \ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1 } \ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ if ($$2 == $$4) files[d] = files[d] " " $$1; \ else { print "f", $$3 "/" $$4, $$1; } } \ END { for (d in files) print "f", d, files[d] }' | \ while read type dir files; do \ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ test -z "$$files" || { \ echo " $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files '$(DESTDIR)$(bindir)$$dir'"; \ $(INSTALL_PROGRAM_ENV) $(LIBTOOL) $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=install $(INSTALL_PROGRAM) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ } \ ; done uninstall-binPROGRAMS: @$(NORMAL_UNINSTALL) @list='$(bin_PROGRAMS)'; test -n "$(bindir)" || list=; \ files=`for p in $$list; do echo "$$p"; done | \ sed -e 'h;s,^.*/,,;s/$(EXEEXT)$$//;$(transform)' \ -e 's/$$/$(EXEEXT)/' \ `; \ test -n "$$list" || exit 0; \ echo " ( cd '$(DESTDIR)$(bindir)' && rm -f" $$files ")"; \ cd "$(DESTDIR)$(bindir)" && rm -f $$files clean-binPROGRAMS: @list='$(bin_PROGRAMS)'; test -n "$$list" || exit 0; \ echo " rm -f" $$list; \ rm -f $$list || exit $$?; \ test -n "$(EXEEXT)" || exit 0; \ list=`for p in $$list; do echo "$$p"; done | sed 's/$(EXEEXT)$$//'`; \ echo " rm -f" $$list; \ rm -f $$list dbadb$(EXEEXT): $(dbadb_OBJECTS) $(dbadb_DEPENDENCIES) $(EXTRA_dbadb_DEPENDENCIES) @rm -f dbadb$(EXEEXT) $(AM_V_CXXLD)$(dbadb_LINK) $(dbadb_OBJECTS) $(dbadb_LDADD) $(LIBS) dbamsg$(EXEEXT): $(dbamsg_OBJECTS) $(dbamsg_DEPENDENCIES) $(EXTRA_dbamsg_DEPENDENCIES) @rm -f dbamsg$(EXEEXT) $(AM_V_CXXLD)$(dbamsg_LINK) $(dbamsg_OBJECTS) $(dbamsg_LDADD) $(LIBS) dbatbl$(EXEEXT): $(dbatbl_OBJECTS) $(dbatbl_DEPENDENCIES) $(EXTRA_dbatbl_DEPENDENCIES) @rm -f dbatbl$(EXEEXT) $(AM_V_CXXLD)$(dbatbl_LINK) $(dbatbl_OBJECTS) $(dbatbl_LDADD) $(LIBS) install-dist_binSCRIPTS: $(dist_bin_SCRIPTS) @$(NORMAL_INSTALL) @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || list=; \ if test -n "$$list"; then \ echo " $(MKDIR_P) '$(DESTDIR)$(bindir)'"; \ $(MKDIR_P) "$(DESTDIR)$(bindir)" || exit 1; \ fi; \ for p in $$list; do \ if test -f "$$p"; then d=; else d="$(srcdir)/"; fi; \ if test -f "$$d$$p"; then echo "$$d$$p"; echo "$$p"; else :; fi; \ done | \ sed -e 'p;s,.*/,,;n' \ -e 'h;s|.*|.|' \ -e 'p;x;s,.*/,,;$(transform)' | sed 'N;N;N;s,\n, ,g' | \ $(AWK) 'BEGIN { files["."] = ""; dirs["."] = 1; } \ { d=$$3; if (dirs[d] != 1) { print "d", d; dirs[d] = 1 } \ if ($$2 == $$4) { files[d] = files[d] " " $$1; \ if (++n[d] == $(am__install_max)) { \ print "f", d, files[d]; n[d] = 0; files[d] = "" } } \ else { print "f", d "/" $$4, $$1 } } \ END { for (d in files) print "f", d, files[d] }' | \ while read type dir files; do \ if test "$$dir" = .; then dir=; else dir=/$$dir; fi; \ test -z "$$files" || { \ echo " $(INSTALL_SCRIPT) $$files '$(DESTDIR)$(bindir)$$dir'"; \ $(INSTALL_SCRIPT) $$files "$(DESTDIR)$(bindir)$$dir" || exit $$?; \ } \ ; done uninstall-dist_binSCRIPTS: @$(NORMAL_UNINSTALL) @list='$(dist_bin_SCRIPTS)'; test -n "$(bindir)" || exit 0; \ files=`for p in $$list; do echo "$$p"; done | \ sed -e 's,.*/,,;$(transform)'`; \ dir='$(DESTDIR)$(bindir)'; $(am__uninstall_files_from_dir) mostlyclean-compile: -rm -f *.$(OBJEXT) distclean-compile: -rm -f *.tab.c @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbadb.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbamsg.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/dbatbl.Po@am__quote@ # am--include-marker $(am__depfiles_remade): @$(MKDIR_P) $(@D) @echo '# dummy' >$@-t && $(am__mv) $@-t $@ am--depfiles: $(am__depfiles_remade) .cc.o: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.o$$||'`;\ @am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ $< .cc.obj: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.obj$$||'`;\ @am__fastdepCXX_TRUE@ $(CXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ `$(CYGPATH_W) '$<'` &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Po @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=no @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(CXXCOMPILE) -c -o $@ `$(CYGPATH_W) '$<'` .cc.lo: @am__fastdepCXX_TRUE@ $(AM_V_CXX)depbase=`echo $@ | sed 's|[^/]*$$|$(DEPDIR)/&|;s|\.lo$$||'`;\ @am__fastdepCXX_TRUE@ $(LTCXXCOMPILE) -MT $@ -MD -MP -MF $$depbase.Tpo -c -o $@ $< &&\ @am__fastdepCXX_TRUE@ $(am__mv) $$depbase.Tpo $$depbase.Plo @AMDEP_TRUE@@am__fastdepCXX_FALSE@ $(AM_V_CXX)source='$<' object='$@' libtool=yes @AMDEPBACKSLASH@ @AMDEP_TRUE@@am__fastdepCXX_FALSE@ DEPDIR=$(DEPDIR) $(CXXDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCXX_FALSE@ $(AM_V_CXX@am__nodep@)$(LTCXXCOMPILE) -c -o $@ $< mostlyclean-libtool: -rm -f *.lo clean-libtool: -rm -rf .libs _libs ID: $(am__tagged_files) $(am__define_uniq_tagged_files); mkid -fID $$unique tags: tags-am TAGS: tags tags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) set x; \ here=`pwd`; \ $(am__define_uniq_tagged_files); \ shift; \ if test -z "$(ETAGS_ARGS)$$*$$unique"; then :; else \ test -n "$$unique" || unique=$$empty_fix; \ if test $$# -gt 0; then \ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ "$$@" $$unique; \ else \ $(ETAGS) $(ETAGSFLAGS) $(AM_ETAGSFLAGS) $(ETAGS_ARGS) \ $$unique; \ fi; \ fi ctags: ctags-am CTAGS: ctags ctags-am: $(TAGS_DEPENDENCIES) $(am__tagged_files) $(am__define_uniq_tagged_files); \ test -z "$(CTAGS_ARGS)$$unique" \ || $(CTAGS) $(CTAGSFLAGS) $(AM_CTAGSFLAGS) $(CTAGS_ARGS) \ $$unique GTAGS: here=`$(am__cd) $(top_builddir) && pwd` \ && $(am__cd) $(top_srcdir) \ && gtags -i $(GTAGS_ARGS) "$$here" cscopelist: cscopelist-am cscopelist-am: $(am__tagged_files) list='$(am__tagged_files)'; \ case "$(srcdir)" in \ [\\/]* | ?:[\\/]*) sdir="$(srcdir)" ;; \ *) sdir=$(subdir)/$(srcdir) ;; \ esac; \ for i in $$list; do \ if test -f "$$i"; then \ echo "$(subdir)/$$i"; \ else \ echo "$$sdir/$$i"; \ fi; \ done >> $(top_builddir)/cscope.files distclean-tags: -rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags distdir: $(BUILT_SOURCES) $(MAKE) $(AM_MAKEFLAGS) distdir-am distdir-am: $(DISTFILES) @srcdirstrip=`echo "$(srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ topsrcdirstrip=`echo "$(top_srcdir)" | sed 's/[].[^$$\\*]/\\\\&/g'`; \ list='$(DISTFILES)'; \ dist_files=`for file in $$list; do echo $$file; done | \ sed -e "s|^$$srcdirstrip/||;t" \ -e "s|^$$topsrcdirstrip/|$(top_builddir)/|;t"`; \ case $$dist_files in \ */*) $(MKDIR_P) `echo "$$dist_files" | \ sed '/\//!d;s|^|$(distdir)/|;s,/[^/]*$$,,' | \ sort -u` ;; \ esac; \ for file in $$dist_files; do \ if test -f $$file || test -d $$file; then d=.; else d=$(srcdir); fi; \ if test -d $$d/$$file; then \ dir=`echo "/$$file" | sed -e 's,/[^/]*$$,,'`; \ if test -d "$(distdir)/$$file"; then \ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ fi; \ if test -d $(srcdir)/$$file && test $$d != $(srcdir); then \ cp -fpR $(srcdir)/$$file "$(distdir)$$dir" || exit 1; \ find "$(distdir)/$$file" -type d ! -perm -700 -exec chmod u+rwx {} \;; \ fi; \ cp -fpR $$d/$$file "$(distdir)$$dir" || exit 1; \ else \ test -f "$(distdir)/$$file" \ || cp -p $$d/$$file "$(distdir)/$$file" \ || exit 1; \ fi; \ done check-am: all-am check: check-am all-am: Makefile $(PROGRAMS) $(SCRIPTS) installdirs: for dir in "$(DESTDIR)$(bindir)" "$(DESTDIR)$(bindir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ done install: install-am install-exec: install-exec-am install-data: install-data-am uninstall: uninstall-am install-am: all-am @$(MAKE) $(AM_MAKEFLAGS) install-exec-am install-data-am installcheck: installcheck-am install-strip: if test -z '$(STRIP)'; then \ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ install; \ else \ $(MAKE) $(AM_MAKEFLAGS) INSTALL_PROGRAM="$(INSTALL_STRIP_PROGRAM)" \ install_sh_PROGRAM="$(INSTALL_STRIP_PROGRAM)" INSTALL_STRIP_FLAG=-s \ "INSTALL_PROGRAM_ENV=STRIPPROG='$(STRIP)'" install; \ fi mostlyclean-generic: clean-generic: distclean-generic: -test -z "$(CONFIG_CLEAN_FILES)" || rm -f $(CONFIG_CLEAN_FILES) -test . = "$(srcdir)" || test -z "$(CONFIG_CLEAN_VPATH_FILES)" || rm -f $(CONFIG_CLEAN_VPATH_FILES) maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." clean: clean-am clean-am: clean-binPROGRAMS clean-generic clean-libtool mostlyclean-am distclean: distclean-am -rm -f ./$(DEPDIR)/dbadb.Po -rm -f ./$(DEPDIR)/dbamsg.Po -rm -f ./$(DEPDIR)/dbatbl.Po -rm -f Makefile distclean-am: clean-am distclean-compile distclean-generic \ distclean-tags dvi: dvi-am dvi-am: html: html-am html-am: info: info-am info-am: install-data-am: install-dvi: install-dvi-am install-dvi-am: install-exec-am: install-binPROGRAMS install-dist_binSCRIPTS install-html: install-html-am install-html-am: install-info: install-info-am install-info-am: install-man: install-pdf: install-pdf-am install-pdf-am: install-ps: install-ps-am install-ps-am: installcheck-am: maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/dbadb.Po -rm -f ./$(DEPDIR)/dbamsg.Po -rm -f ./$(DEPDIR)/dbatbl.Po -rm -f Makefile maintainer-clean-am: distclean-am maintainer-clean-generic mostlyclean: mostlyclean-am mostlyclean-am: mostlyclean-compile mostlyclean-generic \ mostlyclean-libtool pdf: pdf-am pdf-am: ps: ps-am ps-am: uninstall-am: uninstall-binPROGRAMS uninstall-dist_binSCRIPTS .MAKE: install-am install-strip .PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ clean-binPROGRAMS clean-generic clean-libtool cscopelist-am \ ctags ctags-am distclean distclean-compile distclean-generic \ distclean-libtool distclean-tags distdir dvi dvi-am html \ html-am info info-am install install-am install-binPROGRAMS \ install-data install-data-am install-dist_binSCRIPTS \ install-dvi install-dvi-am install-exec install-exec-am \ install-html install-html-am install-info install-info-am \ install-man install-pdf install-pdf-am install-ps \ install-ps-am install-strip installcheck installcheck-am \ installdirs maintainer-clean maintainer-clean-generic \ mostlyclean mostlyclean-compile mostlyclean-generic \ mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ uninstall-am uninstall-binPROGRAMS uninstall-dist_binSCRIPTS .PRECIOUS: Makefile # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: dballe-8.6/src/dbamsg.cc0000644000175000017500000012252513574457765012124 00000000000000/* For %zd */ #define _ISOC99_SOURCE #include "dballe/message.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include "dballe/msg/bulletin.h" #include "dballe/file.h" #include "dballe/core/query.h" #include "dballe/core/matcher.h" #include "dballe/core/csv.h" #include "dballe/core/json.h" #include "dballe/core/var.h" #include "dballe/cmdline/cmdline.h" #include "dballe/cmdline/processor.h" #include "dballe/cmdline/conversion.h" #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include using namespace wreport; using namespace dballe; using namespace dballe::cmdline; using namespace std; static int op_dump_interpreted = 0; static int op_dump_text = 0; static int op_dump_csv = 0; static int op_dump_json = 0; static int op_dump_dds = 0; static int op_dump_structured = 0; static int op_precise_import = 0; static int op_bufr2netcdf_categories = 0; static const char* op_output_type = "bufr"; static const char* op_output_template = ""; static const char* op_output_file = nullptr; static const char* op_report = ""; static const char* op_bisect_cmd = nullptr; int op_verbose = 0; struct cmdline::ReaderOptions readeropts; struct poptOption grepTable[] = { { "category", 0, POPT_ARG_INT, &readeropts.category, 0, "match messages with the given data category", "num" }, { "subcategory", 0, POPT_ARG_INT, &readeropts.subcategory, 0, "match BUFR messages with the given data subcategory", "num" }, { "check-digit", 0, POPT_ARG_INT, &readeropts.checkdigit, 0, "match CREX messages with check digit (if 1) or without check digit (if 0)", "num" }, { "unparsable", 0, 0, &readeropts.unparsable, 0, "match only messages that cannot be parsed", 0 }, { "parsable", 0, 0, &readeropts.parsable, 0, "match only messages that can be parsed", 0 }, { "index", 0, POPT_ARG_STRING, &readeropts.index_filter, 0, "match messages with the index in the given range (ex.: 1-5,9,22-30)", "expr" }, POPT_TABLEEND }; /// Write CSV output to the given output stream struct FileCSV : CSVWriter { FILE* out; FileCSV(FILE* out) : out(out) {} void flush_row() override { fputs(row.c_str(), out); putc('\n', out); row.clear(); } }; volatile int flag_bisect_stop = 0; void stop_bisect(int sig) { /* The signal handler just clears the flag and re-enables itself. */ flag_bisect_stop = 1; signal(sig, stop_bisect); } static int count_nonnulls(const Subset& raw) { unsigned i, count = 0; for (i = 0; i < raw.size(); i++) if (raw[i].isset()) count++; return count; } static void dump_common_header(const BinaryMessage& rmsg, const Bulletin& braw) { printf("Message %d\n", rmsg.index); printf("Size: %zd\n", rmsg.data.size()); printf("Master table number: %hhu\n", braw.master_table_number); printf("Origin: %hu:%hu\n", braw.originating_centre, braw.originating_subcentre); printf("Category: %hhu:%hhu:%hhu\n", braw.data_category, braw.data_subcategory, braw.data_subcategory_local); printf("Update sequence number: %hhu\n", braw.update_sequence_number); printf("Datetime: %04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu\n", braw.rep_year, braw.rep_month, braw.rep_day, braw.rep_hour, braw.rep_minute, braw.rep_second); printf("B Table: %s\n", braw.tables.btable ? braw.tables.btable->pathname().c_str() : "(none)"); printf("D Table: %s\n", braw.tables.dtable ? braw.tables.dtable->pathname().c_str() : "(none)"); } static void dump_bufr_header(const BinaryMessage& rmsg, const BufrBulletin& braw) { dump_common_header(rmsg, braw); printf("BUFR edition: %hhu\n", braw.edition_number); printf("Table version: %hhu:%hhu\n", braw.master_table_version_number, braw.master_table_version_number_local); printf("Compression: %s\n", braw.compression ? "yes" : "no"); printf("Optional section length: %zd\n", braw.optional_section.size()); printf("Subsets: %zd\n\n", braw.subsets.size()); } static void dump_crex_header(const BinaryMessage& rmsg, const CrexBulletin& braw) { dump_common_header(rmsg, braw); printf("CREX edition: %hhu\n", braw.edition_number); printf("Table version: %hhu/%hhu:%hhu\n", braw.master_table_version_number, braw.master_table_version_number_bufr, braw.master_table_version_number_local); printf("Check digit: %s\n\n", braw.has_check_digit ? "yes" : "no"); } static void print_bulletin_header(const Bulletin& braw) { printf(", origin %hu:%hu, category %hhu %hhu:%hhu:%hhu", braw.originating_centre, braw.originating_subcentre, braw.master_table_number, braw.data_category, braw.data_subcategory, braw.data_subcategory_local); } static void print_bufr_header(const BufrBulletin& braw) { print_bulletin_header(braw); printf(", bufr edition %hhu, tables %hhu:%hhu", braw.edition_number, braw.master_table_version_number, braw.master_table_version_number_local); printf(", subsets %zd, values:", braw.subsets.size()); for (size_t i = 0; i < braw.subsets.size(); ++i) printf(" %d/%zd", count_nonnulls(braw.subsets[i]), braw.subsets[i].size()); } static void print_crex_header(const CrexBulletin& braw) { print_bulletin_header(braw); printf(", crex edition %hhu, tables %hhu/%hhu:%hhu", braw.edition_number, braw.master_table_version_number, braw.master_table_version_number_bufr, braw.master_table_version_number_local); printf(", subsets %zd, values:", braw.subsets.size()); for (size_t i = 0; i < braw.subsets.size(); ++i) printf(" %d/%zd", count_nonnulls(braw.subsets[i]), braw.subsets[i].size()); } static void print_item_header(const Item& item) { printf("#%d", item.idx); if (item.rmsg) { printf(" %s message: %zd bytes", File::encoding_name(item.rmsg->encoding), item.rmsg->data.size()); switch (item.rmsg->encoding) { case Encoding::BUFR: if (item.bulletin != NULL) print_bufr_header(*dynamic_cast(item.bulletin)); break; case Encoding::CREX: if (item.bulletin != NULL) print_crex_header(*dynamic_cast(item.bulletin)); break; } } else if (item.msgs) { printf(" message: %zd subsets:", item.msgs->size()); string old_type; unsigned count = 0; for (const auto& i: *item.msgs) { auto m = impl::Message::downcast(i); string new_type = format_message_type(m->type); if (old_type.empty()) { old_type = new_type; count = 1; } else if (old_type != new_type) { printf(" %u %s", count, old_type.c_str()); old_type = new_type; count = 1; } else ++count; } printf(" %u %s", count, old_type.c_str()); } } struct Summarise : public cmdline::Action { virtual bool operator()(const cmdline::Item& item) { print_item_header(item); puts("."); return true; } }; struct Head : public cmdline::Action { virtual bool operator()(const cmdline::Item& item) { if (!item.rmsg) return false; switch (item.rmsg->encoding) { case Encoding::BUFR: if (item.bulletin == NULL) return true; dump_bufr_header(*item.rmsg, *dynamic_cast(item.bulletin)); puts("."); break; case Encoding::CREX: if (item.bulletin == NULL) return true; dump_crex_header(*item.rmsg, *dynamic_cast(item.bulletin)); puts("."); break; } return true; } }; static void dump_dba_vars(const Subset& msg) { for (size_t i = 0; i < msg.size(); ++i) msg[i].print(stdout); } /** * Print a bulletin in CSV format */ struct CSVBulletin : public cmdline::Action { msg::BulletinCSVWriter writer; CSVBulletin() : writer(stdout) {} virtual bool operator()(const cmdline::Item& item) { if (!item.rmsg) return false; if (!item.bulletin) return false; writer.output_bulletin(*item.bulletin); return true; } }; /** * Print a impl::Message in CSV format */ struct CSVMsgs : public cmdline::Action { bool first; FileCSV writer; CSVMsgs() : first(true), writer(stdout) {} virtual bool operator()(const cmdline::Item& item) { if (!item.msgs) return false; if (first) { impl::Message::csv_header(writer); first = false; } for (const auto& mi: *item.msgs) impl::Message::downcast(mi)->to_csv(writer); return true; } }; /** * Print a Msgs in JSON format */ struct JSONMsgs : public cmdline::Action { core::JSONWriter json; JSONMsgs() : json(cout) {} ~JSONMsgs() { cout << flush; } virtual bool operator()(const cmdline::Item& item) { if (!item.msgs) return false; for (const auto& mi: *item.msgs) { auto msg = impl::Message::downcast(mi); json.start_mapping(); json.add("version"); json.add(DBALLE_JSON_VERSION); json.add("network"); json.add(msg->get_rep_memo_var() ? msg->get_rep_memo_var()->enqc() : dballe::impl::Message::repmemo_from_type(msg->type)); json.add("ident"); if (msg->get_ident_var() != NULL) json.add(msg->get_ident_var()->enqc()); else json.add_null(); json.add("lon"); json.add_int(msg->get_longitude_var()->enqi()); json.add("lat"); json.add_int(msg->get_latitude_var()->enqi()); json.add("date"); std::stringstream ss; msg->get_datetime().to_stream_iso8601(ss, 'T', "Z"); json.add(ss.str().c_str()); json.add("data"); json.start_list(); json.start_mapping(); json.add("vars"); json.add(msg->station_data); json.end_mapping(); for (const auto& ctx: msg->data) { json.start_mapping(); json.add("timerange"); json.add(ctx.trange); json.add("level"); json.add(ctx.level); json.add("vars"); json.add(ctx.values); json.end_mapping(); } json.end_list(); json.end_mapping(); json.add_break(); } return true; } }; struct DumpMessage : public cmdline::Action { void print_subsets(const Bulletin& braw) { for (size_t i = 0; i < braw.subsets.size(); ++i) { printf("Subset %zd:\n", i); dump_dba_vars(braw.subsets[i]); } } virtual bool operator()(const cmdline::Item& item) { print_item_header(item); if (!item.rmsg) { puts(": no low-level information available"); return true; } puts(":"); switch (item.rmsg->encoding) { case Encoding::BUFR: { if (item.bulletin == NULL) return true; print_subsets(*item.bulletin); break; } case Encoding::CREX: { if (item.bulletin == NULL) return true; print_subsets(*item.bulletin); break; } } return true; } }; struct DumpCooked : public cmdline::Action { virtual bool operator()(const cmdline::Item& item) { if (item.msgs == NULL) return false; for (size_t i = 0; i < item.msgs->size(); ++i) { printf("#%d[%zd] ", item.idx, i); (*item.msgs)[i]->print(stdout); } return true; } }; static void print_var(const Var& var) { string formatted = var.format(""); printf("%01d%02d%03d %s\n", WR_VAR_FXY(var.code()), formatted.c_str()); } struct DumpText : public cmdline::Action { void add_keyval(const char* key, unsigned val) { printf("%s: %u\n", key, val); } void add_keyval(const char* key, const std::string& val) { printf("%s: %s\n", key, val.c_str()); } virtual bool operator()(const cmdline::Item& item) { if (item.bulletin == NULL) throw error_consistency("source is not a BUFR or CREX message"); const Bulletin& bul = *item.bulletin; add_keyval("master_table_number", bul.master_table_number); add_keyval("data_category", bul.data_category); add_keyval("data_subcategory", bul.data_subcategory); add_keyval("data_subcategory_local", bul.data_subcategory_local); add_keyval("originating_centre", bul.originating_centre); add_keyval("originating_subcentre", bul.originating_subcentre); add_keyval("update_sequence_number", bul.update_sequence_number); char buf[30]; snprintf(buf, 29, "%hu-%hhu-%hhu %hhu:%hhu:%hhu", bul.rep_year, bul.rep_month, bul.rep_day, bul.rep_hour, bul.rep_minute, bul.rep_second); add_keyval("representative_time", buf); if (const BufrBulletin* b = dynamic_cast(item.bulletin)) { add_keyval("encoding", "bufr"); add_keyval("edition_number", b->edition_number); add_keyval("master_table_version_number", b->master_table_version_number); add_keyval("master_table_version_number_local", b->master_table_version_number_local); add_keyval("compression", b->compression ? "true" : "false"); add_keyval("optional_section", b->optional_section); } else if (const CrexBulletin* b = dynamic_cast(item.bulletin)) { add_keyval("encoding", "crex"); add_keyval("edition_number", b->edition_number); add_keyval("master_table_version_number", b->master_table_version_number); add_keyval("master_table_version_number_bufr", b->master_table_version_number_bufr); add_keyval("master_table_version_number_local", b->master_table_version_number_local); add_keyval("has_check_digit", b->has_check_digit ? "true" : "false"); } else throw error_consistency("encoding not supported for CSV dump"); printf("descriptors:"); for (const auto& desc: bul.datadesc) printf(" %01d%02d%03d", WR_VAR_FXY(desc)); printf("\n"); for (size_t i = 0; i < bul.subsets.size(); ++i) { const Subset& subset = bul.subsets[i]; printf("subset %zd:\n", i + 1); for (size_t j = 0; j < subset.size(); ++j) { const Var& var = subset[j]; printf(" "); print_var(var); for (const Var* attr = var.next_attr(); attr; attr = attr->next_attr()) { printf(" *"); print_var(*attr); } } } return true; } }; struct DumpStructured : public cmdline::Action { virtual bool operator()(const cmdline::Item& item) { print_item_header(item); if (!item.rmsg) { puts(": no low-level information available"); return true; } if (!item.bulletin) { puts(": no bulletin information available"); return true; } puts(":"); item.bulletin->print_structured(stdout); return true; } }; struct DumpDDS : public cmdline::Action { virtual bool operator()(const cmdline::Item& item) { print_item_header(item); if (!item.rmsg) { puts(": no low-level information available"); return true; } if (!item.bulletin) { puts(": no bulletin information available"); return true; } puts(":"); item.bulletin->print_datadesc(stdout); return false; } }; struct WriteRaw : public cmdline::Action { File* file; WriteRaw() : file(0) {} ~WriteRaw() { if (file) delete file; } virtual bool operator()(const cmdline::Item& item) { if (!item.rmsg) return false; if (!file) file = File::create(item.rmsg->encoding, stdout, false, "(stdout)").release(); file->write(item.rmsg->data); return true; } }; struct Scan : public cmdline::Subcommand { Scan() { names.push_back("scan"); usage = "scan [options] [filter] filename [filename [...]]"; desc = "Summarise the contents of a file with meteorological data"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); core::Query query; cmdline::Reader reader(readeropts); if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); Summarise s; reader.read(get_filenames(optCon), s); return 0; } }; struct HeadCmd : public cmdline::Subcommand { HeadCmd() { names.push_back("head"); usage = "head [options] [filter] filename [filename [...]]"; desc = "Dump the contents of the header of a file with meteorological data"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); cmdline::Reader reader(readeropts); core::Query query; if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); Head head; reader.read(get_filenames(optCon), head); return 0; } }; struct Dump : public cmdline::Subcommand { Dump() { names.push_back("dump"); usage = "dump [options] [filter] filename [filename [...]]"; desc = "Dump the contents of a file with meteorological data"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ "interpreted", 0, 0, &op_dump_interpreted, 0, "dump the message as understood by the importer", 0 }); opts.push_back({ "precise", 0, 0, &op_precise_import, 0, "import messages using precise contexts instead of standard ones", 0 }); opts.push_back({ "text", 0, 0, &op_dump_text, 0, "dump as text that can be processed by dbamsg makebufr", 0 }); opts.push_back({ "csv", 0, 0, &op_dump_csv, 0, "dump in machine readable CSV format", 0 }); opts.push_back({ "json", 0, 0, &op_dump_json, 0, "dump in machine readable JSON format", 0 }); opts.push_back({ "dds", 0, 0, &op_dump_dds, 0, "dump structure of data description section", 0 }); opts.push_back({ "structured", 0, 0, &op_dump_structured, 0, "structured dump of the message contents", 0 }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { unique_ptr action; if (op_dump_csv) { if (op_dump_interpreted) action.reset(new CSVMsgs); else action.reset(new CSVBulletin); } else if (op_dump_json) action.reset(new JSONMsgs); else if (op_dump_interpreted) action.reset(new DumpCooked); else if (op_dump_text) action.reset(new DumpText); else if (op_dump_structured) action.reset(new DumpStructured); else if (op_dump_dds) action.reset(new DumpDDS); else action.reset(new DumpMessage); /* Throw away the command name */ poptGetArg(optCon); cmdline::Reader reader(readeropts); if (op_precise_import) reader.import_opts.simplified = false; core::Query query; if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); reader.read(get_filenames(optCon), *action); return 0; } }; struct Cat : public cmdline::Subcommand { Cat() { names.push_back("cat"); usage = "cat [options] [filter] filename [filename [...]]"; desc = "Dump the raw data of a file with meteorological data"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); cmdline::Reader reader(readeropts); core::Query query; if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); WriteRaw wraw; reader.read(get_filenames(optCon), wraw); return 0; } }; #if 0 struct StoreMessages : public cmdline::Action, public vector { void operator()(const BinaryMessage& rmsg, const wreport::Bulletin*, const impl::Messages*) override { push_back(rmsg); } }; #endif #if 0 static dba_err bisect_test(struct message_vector* vec, size_t first, size_t last, int* fails) { FILE* out = popen(op_bisect_cmd, "w"); int res; for (; first < last; ++first) { dba_rawmsg msg = vec->messages[first]; if (fwrite(msg->buf, msg->len, 1, out) == 0) return dba_error_system("writing message %d to test script", msg->index); } res = pclose(out); if (res == -1) return dba_error_system("running test script", first); *fails = (res != 0); return dba_error_ok(); } struct bisect_candidate { size_t first; size_t last; }; static dba_err bisect( struct bisect_candidate* cand, struct message_vector* vec, size_t first, size_t last) { int fails = 0; /* If we already narrowed it down to 1 messages, there is no need to test * further */ if (flag_bisect_stop || cand->last == cand->first + 1) return dba_error_ok(); if (op_verbose) fprintf(stderr, "Trying messages %zd-%zd (%zd selected, kill -HUP %d to stop)... ", first, last, cand->last - cand->first, getpid()); DBA_RUN_OR_RETURN(bisect_test(vec, first, last, &fails)); if (op_verbose) fprintf(stderr, fails ? "fail.\n" : "ok.\n"); if (fails) { size_t mid = (first + last) / 2; if (last-first < cand->last - cand->first) { cand->last = last; cand->first = first; } if (first < mid && mid != last) DBA_RUN_OR_RETURN(bisect(cand, vec, first, mid)); if (mid < last && mid != first) DBA_RUN_OR_RETURN(bisect(cand, vec, mid, last)); } return dba_error_ok(); } #endif struct Bisect : public cmdline::Subcommand { Bisect() { names.push_back("bisect"); usage = "bisect [options] --test=testscript filename"; desc = "Bisect filename and output the minimum subsequence found for which testscript fails."; longdesc = "Run testscript passing parts of filename on its stdin and checking the return code. Then divide the input in half and try on each half. Keep going until testscript does not fail in any portion of the file. Output to stdout the smallest portion for which testscript fails. This is useful to isolate the few messages in a file that cause problems"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "test", 0, POPT_ARG_STRING, &op_bisect_cmd, 0, "command to run to test a message group", "cmd" }); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { #if 0 struct message_vector vec = { 0, 0, 0 }; struct bisect_candidate candidate; int old_op_verbose = op_verbose; size_t i; /* Throw away the command name */ poptGetArg(optCon); if (op_bisect_cmd == NULL) return dba_error_consistency("you need to use --test=command"); /* Read all input messages a vector of dba_rawmsg */ op_verbose = 0; DBA_RUN_OR_RETURN(process_all(optCon, dba_cmdline_stringToMsgType(reader.input_type, optCon), &reader.filter, store_messages, &vec)); op_verbose = old_op_verbose; /* Establish a handler for SIGHUP signals. */ signal(SIGHUP, stop_bisect); /* Bisect working on the vector */ candidate.first = 0; candidate.last = vec.len; DBA_RUN_OR_RETURN(bisect(&candidate, &vec, candidate.first, candidate.last)); if (op_verbose) { if (flag_bisect_stop) fprintf(stderr, "Stopped by SIGHUP.\n"); fprintf(stderr, "Selected messages %zd-%zd.\n", candidate.first, candidate.last); } /* Output the candidate messages */ for (; candidate.first < candidate.last; ++candidate.first) { dba_rawmsg msg = vec.messages[candidate.first]; if (fwrite(msg->buf, msg->len, 1, stdout) == 0) return dba_error_system("writing message %d to standard output", msg->index); } for (i = 0; i < vec.len; ++i) dba_rawmsg_delete(vec.messages[i]); free(vec.messages); return dba_error_ok(); #endif throw error_unimplemented("bisect is currently not implemented"); } }; struct Convert : public cmdline::Subcommand { Convert() { names.push_back("convert"); names.push_back("conv"); usage = "convert [options] [filter] filename [filename [...]]"; desc = "Convert meteorological data between different formats"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the input data ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "dest", 'd', POPT_ARG_STRING, &op_output_type, 0, "format of the data in output ('bufr', 'crex')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ "template", 0, POPT_ARG_STRING, &op_output_template, 0, "template of the data in output (autoselect if not specified, 'list' gives a list)", "name" }); opts.push_back({ "report", 'r', POPT_ARG_STRING, &op_report, 0, "force output data to be of this type of report", "rep_memo" }); opts.push_back({ "precise", 0, 0, &op_precise_import, 0, "import messages using precise contexts instead of standard ones", 0 }); opts.push_back({ "bufr2netcdf-categories", 0, 0, &op_bufr2netcdf_categories, 0, "recompute data categories and subcategories according to message contents, for use as input to bufr2netcdf", 0 }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); opts.push_back({ "output", 'o', POPT_ARG_STRING, &op_output_file, 0, "destination file. Default: stdandard output", "fname" }); } int main(poptContext optCon) override { impl::ExporterOptions opts; cmdline::Converter conv; cmdline::Reader reader(readeropts); reader.verbose = op_verbose; /* Throw away the command name */ poptGetArg(optCon); if (strcmp(op_output_template, "list") == 0) { list_templates(); return 0; } core::Query query; if (dba_cmdline_get_query(optCon, query) > 0) reader.filter.matcher_from_record(query); if (op_precise_import) reader.import_opts.simplified = false; if (op_report[0] != 0) conv.dest_rep_memo = op_report; else conv.dest_rep_memo = NULL; if (op_output_template[0] != 0) { conv.dest_template = op_output_template; opts.template_name = op_output_template; } conv.bufr2netcdf_categories = op_bufr2netcdf_categories != 0; if (strcmp(op_output_type, "auto") == 0) { if (op_output_file == NULL) conv.file = File::create(stdout, false, "stdout").release(); else conv.file = File::create(op_output_file, "w").release(); } else { if (op_output_file == NULL) conv.file = File::create(string_to_encoding(op_output_type), stdout, false, "stdout").release(); else conv.file = File::create(string_to_encoding(op_output_type), op_output_file, "w").release(); } conv.exporter = Exporter::create(conv.file->encoding(), opts).release(); reader.read(get_filenames(optCon), conv); return 0; } }; struct Compare : public cmdline::Subcommand { Compare() { names.push_back("compare"); names.push_back("cmp"); usage = "compare [options] filename1 [filename2]"; desc = "Compare two files with meteorological data"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type1", 't', POPT_ARG_STRING, &readeropts.input_type, 0, "format of the first file to compare ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "type2", 'd', POPT_ARG_STRING, &op_output_type, 0, "format of the second file to compare ('bufr', 'crex', 'json', 'csv')", "type" }); opts.push_back({ "rejected", 0, POPT_ARG_STRING, &readeropts.fail_file_name, 0, "write unprocessed data to this file", "fname" }); opts.push_back({ NULL, 0, POPT_ARG_INCLUDE_TABLE, &grepTable, 0, "Options used to filter messages", 0 }); } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); cmdline::Reader reader(readeropts); /* Read the file names */ const char* file1_name = poptGetArg(optCon); if (file1_name == NULL) dba_cmdline_error(optCon, "input file needs to be specified"); const char* file2_name = poptGetArg(optCon); if (file2_name == NULL) file2_name = "(stdin)"; unique_ptr file1; unique_ptr file2; if (strcmp(readeropts.input_type, "auto") == 0) file1 = File::create(file1_name, "r"); else file1 = File::create(string_to_encoding(readeropts.input_type), file1_name, "r"); if (strcmp(op_output_type, "auto") == 0) file2 = File::create(file2_name, "r"); else file2 = File::create(string_to_encoding(op_output_type), file2_name, "r"); std::unique_ptr importer1 = Importer::create(file1->encoding()); std::unique_ptr importer2 = Importer::create(file2->encoding()); size_t idx = 0; for ( ; ; ++idx) { ++idx; BinaryMessage msg1 = file1->read(); BinaryMessage msg2 = file2->read(); bool found1 = msg1; bool found2 = msg2; if (found1 != found2) throw error_consistency("The files contain a different number of messages"); if (!found1 && !found2) break; impl::Messages msgs1 = importer1->from_binary(msg1); impl::Messages msgs2 = importer2->from_binary(msg2); notes::Collect c(cerr); int diffs = impl::msg::messages_diff(msgs1, msgs2); if (diffs > 0) error_consistency::throwf("Messages #%zd contain %d differences", idx, diffs); } if (idx == 0) throw error_consistency("The files do not contain messages"); return 0; } }; #if 0 static dba_err readfield(FILE* in, char** name, char** value) { static char line[1000]; char* s; if (fgets(line, 1000, in) == NULL) { *name = *value = NULL; return dba_error_ok(); } s = strchr(line, ':'); if (s == NULL) { *name = NULL; *value = line; } else { *s = 0; *name = line; *value = s + 1; } if (*value) { int len; /* Trim value */ while (**value && isspace(**value)) ++*value; len = strlen(*value); while (len > 0 && isspace((*value)[len-1])) { --len; (*value)[len] = 0; } } return dba_error_ok(); } #endif #if 0 static dba_err parsetextgrib(FILE* in, bufrex_msg msg, int* found) { dba_err err = DBA_OK; bufrex_subset subset = NULL; char* name; char* value; dba_var var = NULL; *found = 0; bufrex_msg_reset(msg); while (1) { DBA_RUN_OR_GOTO(cleanup, readfield(in, &name, &value)); /* fprintf(stderr, "GOT NAME %s VALUE \"%s\"\n", name, value); */ if (name != NULL) { if (strcasecmp(name, "edition") == 0) { msg->edition = strtoul(value, 0, 10); } else if (strcasecmp(name, "type") == 0) { msg->type = strtoul(value, 0, 10); } else if (strcasecmp(name, "subtype") == 0) { msg->subtype = strtoul(value, 0, 10); } else if (strcasecmp(name, "localsubtype") == 0) { msg->localsubtype = strtoul(value, 0, 10); } else if (strcasecmp(name, "centre") == 0) { msg->opt.bufr.centre = strtoul(value, 0, 10); } else if (strcasecmp(name, "subcentre") == 0) { msg->opt.bufr.subcentre = strtoul(value, 0, 10); } else if (strcasecmp(name, "mastertable") == 0) { msg->opt.bufr.master_table = strtoul(value, 0, 10); } else if (strcasecmp(name, "localtable") == 0) { msg->opt.bufr.local_table = strtoul(value, 0, 10); } else if (strcasecmp(name, "compression") == 0) { msg->opt.bufr.compression = strtoul(value, 0, 10); } else if (strcasecmp(name, "reftime") == 0) { if (sscanf(value, "%04d-%02d-%02d %02d:%02d:%02d", &msg->rep_year, &msg->rep_month, &msg->rep_day, &msg->rep_hour, &msg->rep_minute, &msg->rep_second) != 6) return dba_error_consistency("Reference time \"%s\" cannot be parsed", value); } else if (strcasecmp(name, "descriptors") == 0) { const char* s = value; DBA_RUN_OR_GOTO(cleanup, bufrex_msg_load_tables(msg)); while (1) { size_t size = strcspn(s, " \t"); s += size; size = strspn(s, "BCDR0123456789"); if (size == 0) break; else DBA_RUN_OR_GOTO(cleanup, bufrex_msg_append_datadesc(msg, dba_descriptor_code(s))); } } else if (strcasecmp(name, "data") == 0) { /* Start a new subset */ DBA_RUN_OR_GOTO(cleanup, bufrex_msg_get_subset(msg, msg->subsets_count, &subset)); *found = 1; } } else if (value != NULL) { dba_varinfo info; int isattr = 0; dba_varcode code; if (value[0] == 0) /* End of one message */ break; /* Read a Bsomething (value or attribute) and append it to the subset */ if (value[0] == '*') { isattr = 1; ++value; } code = dba_descriptor_code(value); DBA_RUN_OR_GOTO(cleanup, bufrex_msg_query_btable(msg, code, &info)); while (*value && !isspace(*value)) ++value; while (*value && isspace(*value)) ++value; if (*value == 0) { /* Undef */ DBA_RUN_OR_GOTO(cleanup, dba_var_create(info, &var)); } else { if (VARINFO_IS_STRING(info)) { DBA_RUN_OR_GOTO(cleanup, dba_var_createc(info, value, &var)); } else { DBA_RUN_OR_GOTO(cleanup, dba_var_created(info, strtod(value, NULL), &var)); } } if (isattr) { DBA_RUN_OR_GOTO(cleanup, bufrex_subset_add_attr(subset, var)); dba_var_delete(var); } else { DBA_RUN_OR_GOTO(cleanup, bufrex_subset_store_variable(subset, var)); } var = NULL; } else { /* End of input */ break; } } cleanup: if (var) dba_var_delete(var); return err == DBA_OK ? dba_error_ok() : err; } #endif struct MakeBUFR : public cmdline::Subcommand { MakeBUFR() { names.push_back("makebufr"); names.push_back("mkbufr"); usage = "makebufr [options] filename [filename1 [...]]]"; desc = "Read a simple description of a BUFR file and output the BUFR file."; longdesc = "Read a simple description of a BUFR file and output the BUFR file. This only works for simple BUFR messages without attributes encoded with data present bitmaps"; } int main(poptContext optCon) override { throw error_unimplemented("makebufr not implemented"); #if 0 dba_err err = DBA_OK; bufrex_msg msg = NULL; dba_rawmsg rmsg = NULL; dba_file outfile = NULL; const char* filename; FILE* in = NULL; int count = 0; DBA_RUN_OR_RETURN(bufrex_msg_create(BUFREX_BUFR, &msg)); DBA_RUN_OR_GOTO(cleanup, dba_file_create(BUFR, "(stdout)", "w", &outfile)); /* Throw away the command name */ poptGetArg(optCon); while ((filename = poptGetArg(optCon)) != NULL) { int found; in = fopen(filename, "r"); if (in == NULL) { err = dba_error_system("opening file %s", filename); goto cleanup; } while (1) { DBA_RUN_OR_GOTO(cleanup, parsetextgrib(in, msg, &found)); if (found) { DBA_RUN_OR_GOTO(cleanup, bufrex_msg_encode(msg, &rmsg)); DBA_RUN_OR_GOTO(cleanup, dba_file_write(outfile, rmsg)); dba_rawmsg_delete(rmsg); rmsg = NULL; } else break; } fclose(in); in = NULL; ++count; } if (count == 0) dba_cmdline_error(optCon, "at least one input file needs to be specified"); cleanup: if (in != NULL) fclose(in); if (msg) bufrex_msg_delete(msg); if (rmsg) dba_rawmsg_delete(rmsg); if (outfile) dba_file_delete(outfile); return err == DBA_OK ? dba_error_ok() : err; #endif return 0; } }; int main (int argc, const char* argv[]) { Command dbamsg; dbamsg.name = "dbamsg"; dbamsg.desc = "Work with encoded meteorological data"; dbamsg.longdesc = "Examine, dump and convert files containing meteorological data. " "It supports observations encoded in BUFR or CREX formats"; dbamsg.manpage_examples_section = R"( Here are some example invocations of \\fBdbamsg\\fP: .P .nf # Convert a BUFR message to CREX dbamsg convert file.bufr -d crex > file.crex # Convert BUFR messages to CREX, but skip all those not in january 2010 dbamsg convert year=2010 month=1 file.bufr -d crex > file.crex # Dump the content of a message, as they are in the message dbamsg dump file.bufr # Dump the content of a message, interpreted as physical quantities dbamsg dump --interpreted file.bufr .fi )"; dbamsg.add_subcommand(new Scan); dbamsg.add_subcommand(new HeadCmd); dbamsg.add_subcommand(new Dump); dbamsg.add_subcommand(new Cat); dbamsg.add_subcommand(new Bisect); dbamsg.add_subcommand(new Convert); dbamsg.add_subcommand(new Compare); dbamsg.add_subcommand(new MakeBUFR); return dbamsg.main(argc, argv); } dballe-8.6/src/dbatbl.cc0000644000175000017500000003413013554564112012070 00000000000000#include #include #include #include #include #include #include #include #include #include "config.h" using namespace dballe; using namespace dballe::cmdline; using namespace wreport; using namespace std; int op_verbose = 0; /// Write CSV output to the given output stream struct FileCSV : CSVWriter { FILE* out; FileCSV(FILE* out) : out(out) {} void flush_row() override { fputs(row.c_str(), out); putc('\n', out); row.clear(); } }; struct VarinfoPrinter : public cmdline::Subcommand { int op_csv = 0; int op_crex = 0; CSVWriter* csv_out = nullptr; ~VarinfoPrinter() { delete csv_out; } void init() override { if (op_csv) csv_out = new FileCSV(stdout); } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back(poptOption{ "csv", 'c', POPT_ARG_NONE, (void*)&op_csv, 0, "output variables in CSV format" }); opts.push_back(poptOption{ "crex", 0, POPT_ARG_NONE, (void*)&op_crex, 0, "read CREX entries instead of BUFR" }); } const Vartable* load_vartable(const char* id) { if (op_crex) return Vartable::load_crex(id); else return Vartable::load_bufr(id); } void print_varinfo(Varinfo info) const { if (csv_out) print_varinfo_csv(*csv_out, info); else print_varinfo_desc(info); } static std::string format_decimal(Varinfo info) { string res; if (info->scale > 0) { if (info->len > (unsigned)info->scale) for (unsigned i = 0; i < info->len - info->scale; ++i) res += '#'; res += '.'; for (unsigned i = 0; i < (unsigned)info->scale; ++i) res += '#'; } else if (info->scale < 0) { for (unsigned i = 0; i < info->len; ++i) res += '#'; for (unsigned i = 0; i < (unsigned)-info->scale; ++i) res += '0'; } return res; } static void print_varinfo_desc(Varinfo info) { switch (info->type) { case Vartype::String: printf("%d%02d%03d %s [%s, %u characters]\n", WR_VAR_FXY(info->code), info->desc, info->unit, info->len); break; case Vartype::Binary: printf("%d%02d%03d %s [%s, %u bits]\n", WR_VAR_FXY(info->code), info->desc, info->unit, info->bit_len); break; case Vartype::Integer: printf("%d%02d%03d %s [%s, %u digits] range (%d -- %d)\n", WR_VAR_FXY(info->code), info->desc, info->unit, info->len, info->imin, info->imax); break; case Vartype::Decimal: printf("%d%02d%03d %s [%s, %s] range (%g -- %g)\n", WR_VAR_FXY(info->code), info->desc, info->unit, format_decimal(info).c_str(), info->dmin, info->dmax); break; } } static void print_varinfo_csv(CSVWriter& out, const Varinfo& info) { out.add_value(varcode_format(info->code)); out.add_value(info->desc); out.add_value(info->unit); out.add_value(vartype_format(info->type)); switch (info->type) { case Vartype::String: out.add_value(info->len); out.add_value_empty(); out.add_value_empty(); break; case Vartype::Binary: out.add_value(info->bit_len); out.add_value_empty(); out.add_value_empty(); break; case Vartype::Integer: out.add_value(info->len); out.add_value(info->imin); out.add_value(info->imax); break; case Vartype::Decimal: out.add_value(format_decimal(info)); out.add_value(to_string(info->dmin)); out.add_value(to_string(info->dmax)); break; } out.flush_row(); } }; struct Cat : public VarinfoPrinter { Cat() { names.push_back("cat"); usage = "cat tableid [tableid [...]]"; desc = "Output all the contents of a WMO B table."; } int main(poptContext optCon) override { const char* item; /* Throw away the command name */ poptGetArg(optCon); if (poptPeekArg(optCon) == NULL) item = "dballe"; else item = poptGetArg(optCon); while (item != NULL) { const Vartable* table = load_vartable(item); table->iterate([&](Varinfo info) { print_varinfo(info); return true; }); item = poptGetArg(optCon); } return 0; } }; struct Grep : public VarinfoPrinter { Grep() { names.push_back("grep"); usage = "grep string"; desc = "Output all the contents of the local B table whose description contains the given string."; } int main(poptContext optCon) override { /* Throw away the command name */ poptGetArg(optCon); if (poptPeekArg(optCon) == NULL) dba_cmdline_error(optCon, "there should be at least one B or D item to expand. Examples are: B01002 or D03001"); const char* pattern = poptGetArg(optCon); const Vartable* table = load_vartable("dballe"); table->iterate([&](Varinfo info) { #if HAVE_STRCASESTR if (strcasestr(info->desc, pattern) != NULL) #else #warning dbatbl grep is case sensite on this sytstem, since strcasestr is not available if (strstr(info->desc, pattern) != NULL) #endif print_varinfo(info); return true; }); return 0; } }; #if 0 struct Expand : public cmdline::Subcommand { const Vartable* btable = NULL; const DTable* dtable = NULL; Expand() { names.push_back("expand"); usage = "expand table-entry [table-entry [...]]"; desc = "Describe a WMO B table entry or expand a WMO D table entry in its components."; } int main(poptContext optCon) override { const char* item; /* Throw away the command name */ poptGetArg(optCon); btable = Vartable::get("dballe"); dtable = DTable::get("D000203"); if (poptPeekArg(optCon) == NULL) dba_cmdline_error(optCon, "there should be at least one B or D item to expand. Examples are: B01002 or D03001"); while ((item = poptGetArg(optCon)) != NULL) expand_table_entry(descriptor_code(item), 0); return 0; } void expand_table_entry(Varcode val, int level) { int i; for (i = 0; i < level; i++) printf("\t"); switch (WR_VAR_F(val)) { case 0: { Varinfo info = btable->query(val); VarinfoPrinter::print_varinfo_desc(info); break; } case 3: { Opcodes ops = dtable->query(val); printf("%d%02d%03d\n", WR_VAR_F(val), WR_VAR_X(val), WR_VAR_Y(val)); for (size_t i = 0; i < ops.size(); ++i) expand_table_entry(ops[i], level+1); break; } default: printf("%d%02d%03d\n", WR_VAR_F(val), WR_VAR_X(val), WR_VAR_Y(val)); } } }; #endif struct ExpandCode : public cmdline::Subcommand { ExpandCode() { names.push_back("expandcode"); usage = "expandcode varcode [varcode [...]]"; desc = "Expand the value of a packed variable code"; } int main(poptContext optCon) override { const char* item; /* Throw away the command name */ poptGetArg(optCon); while ((item = poptGetArg(optCon)) != NULL) { int code = strtol(item, NULL, 10); char c = 'B'; switch (WR_VAR_F(code)) { case 0: c = 'B'; break; case 1: c = 'R'; break; case 2: c = 'C'; break; case 3: c = 'D'; break; } printf("%s: %c%02d%03d\n", item, c, WR_VAR_X(code), WR_VAR_Y(code)); } return 0; } }; #if 0 static const char* table_type = "b"; struct Index : public cmdline::Subcommand { Index() { names.push_back("index"); usage = "index [options] filename index-id"; desc = "Index the contents of a table file"; } void add_to_optable(std::vector& opts) const override { Subcommand::add_to_optable(opts); opts.push_back({ "type", 't', POPT_ARG_STRING, &table_type, 0, "format of the table to index ('b', 'd', 'conv')", "type" }); } /** * Check that all unit conversions are allowed by dba_uniconv * * @returns true if all conversions worked, false if some exceptions were thrown */ static bool check_unit_conversions(const char* id) { const Vartable* othertable = Vartable::get(id); bool res = true; for (Vartable::const_iterator info = othertable->begin(); info != othertable->end(); ++info) { Varcode varcode = info->var; /* if (var % 1000 == 0) fprintf(stderr, "Testing %s %d\n", ids[i], var); */ if (varcode != 0 && !info->is_string()) { try { Varinfo local = varinfo(varcode); convert_units(info->unit, local->unit, 1.0); } catch (std::exception& e) { fprintf(stderr, "Checking conversion for var B%02d%03d: %s", WR_VAR_X(varcode), WR_VAR_Y(varcode), e.what()); res = false; } } } return res; } int main(poptContext optCon) override { const char* file; const char* id; /* Throw away the command name */ poptGetArg(optCon); file = poptGetArg(optCon); id = poptGetArg(optCon); if (file == NULL) dba_cmdline_error(optCon, "input file has not been specified"); if (id == NULL) dba_cmdline_error(optCon, "indexed table ID has not been specified"); if (strcmp(table_type, "b") == 0) { if (strcmp(id, "dballe") != 0) { /* If it's an external table, check unit conversions to DBALLE * correspondents */ if (!check_unit_conversions(id)) fprintf(stderr, "Warning: some variables cannot be converted from %s to dballe\n", id); } } else if (strcmp(table_type, "d") == 0) ; /* DBA_RUN_OR_RETURN(bufrex_dtable_index(file, id)); */ /* else if (strcmp(type, "conv") == 0) DBA_RUN_OR_RETURN(bufrex_convtable_index_csv(file, id)); */ else dba_cmdline_error(optCon, "'%s' is not a valid table type", table_type); return 0; } }; #endif struct Describe : public cmdline::Subcommand { Describe() { names.push_back("describe"); usage = "describe [options] what [values]"; desc = "Invoke the formatter to describe the given values"; longdesc = "Supported so far are: \"level ltype l1 l2\", \"trange pind p1 p2\""; } int main(poptContext optCon) override { const char* what; /* Throw away the command name */ poptGetArg(optCon); if ((what = poptGetArg(optCon)) == NULL) dba_cmdline_error(optCon, "you need to specify what you want to describe. Available options are: 'level' and 'trange'"); if (strcmp(what, "level") == 0) { const char* sltype1 = poptGetArg(optCon); const char* sl1 = poptGetArg(optCon); const char* sltype2 = poptGetArg(optCon); const char* sl2 = poptGetArg(optCon); if (sltype1 == NULL) dba_cmdline_error(optCon, "you need provide 1, 2, 3 or 4 numbers that identify the level or layer"); string formatted = Level( strtoul(sltype1, NULL, 10), sl1 == NULL ? 0 : strtoul(sl1, NULL, 10), sltype2 == NULL ? 0 : strtoul(sltype2, NULL, 10), sl2 == NULL ? 0 : strtoul(sl2, NULL, 10)).describe(); puts(formatted.c_str()); } else if (strcmp(what, "trange") == 0) { const char* sptype = poptGetArg(optCon); const char* sp1 = poptGetArg(optCon); const char* sp2 = poptGetArg(optCon); if (sptype == NULL) dba_cmdline_error(optCon, "you need provide 1, 2 or 3 numbers that identify the time range"); string formatted = Trange( strtoul(sptype, NULL, 10), sp1 == NULL ? 0 : strtoul(sp1, NULL, 10), sp2 == NULL ? 0 : strtoul(sp2, NULL, 10)).describe(); puts(formatted.c_str()); } else dba_cmdline_error(optCon, "cannot handle %s. Available options are: 'level' and 'trange'.", what); return 0; } }; int main(int argc, const char* argv[]) { Command dbatbl; dbatbl.name = "dbatbl"; dbatbl.desc = "Manage on-disk reference tables for DB-ALLe"; dbatbl.longdesc = "This tool allows to index and query the tables that are " "needed for normal functioning of DB-ALLe"; dbatbl.add_subcommand(new Cat); dbatbl.add_subcommand(new Grep); //dbatbl.add_subcommand(new Expand); dbatbl.add_subcommand(new ExpandCode); //dbatbl.add_subcommand(new Index); dbatbl.add_subcommand(new Describe); return dbatbl.main(argc, argv); } dballe-8.6/autogen.sh0000755000175000017500000000006513554564112011543 00000000000000#!/bin/sh # Rebuild the build system autoreconf -i dballe-8.6/README.md0000644000175000017500000001337513554573614011040 00000000000000DB-All.e =============================================================== [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=centos:7&label=centos7)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=centos:8&label=centos8)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=fedora:29&label=fedora29)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=fedora:30&label=fedora30)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=fedora:31&label=fedora31)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://badges.herokuapp.com/travis/ARPA-SIMC/dballe?branch=master&env=DOCKER_IMAGE=fedora:rawhide&label=fedorarawhide)](https://travis-ci.org/ARPA-SIMC/dballe) [![Build Status](https://copr.fedorainfracloud.org/coprs/simc/stable/package/dballe/status_image/last_build.png)](https://copr.fedorainfracloud.org/coprs/simc/stable/package/dballe/) Introduction ------------ DB-All.e is a fast on-disk database where meteorological observed and forecast data can be stored, searched, retrieved and updated. This framework allows to manage large amounts of data using its simple Application Program Interface, and provides tools to visualise, import and export in the standard formats BUFR, and CREX. The main characteristics of DB-ALL.e are: * Fortran, C, C++ and Python APIs are provided. * To make computation easier, data is stored as physical quantities, that is, as measures of a variable in a specific point of space and time, rather than as a sequence of report. * Internal representation is similar to BUFR and CREX WMO standard (table code driven) and utility for import and export are included (generic and ECMWF template). * Representation is in 7 dimensions: latitude and longitude geographic coordinates, table driven vertical coordinate, reference time, table driven observation and forecast specification, table driven data type. * It allows to store extra information linked to the data, such as confidence intervals for quality control. * It allows to store extra information linked to the stations. * Variables can be represented as real, integer and characters, with appropriate precision for the type of measured value. * It is based on physical principles, that is, the data it contains are defined in terms of homogeneous and consistent physical data. For example, it is impossible for two incompatible values to exist in the same point in space and time. * It can manage fixed stations and moving stations such as airplanes or ships. * It can manage both observational and forecast data. * It can manage data along all three dimensions in space, such as data from soundings and airplanes. * Report information is preserved. It can work based on physical parameters or on report types. * It is temporary, to be used for a limited time and then be deleted. * Does not need backup, since it only contains replicated or derived data. * Write access is enabled for its users. [DB-All.e documentation](https://arpa-simc.github.io/dballe/). Building DB-All.e ----------------- DB-All.e is already packaged in both .rpm and .deb formats, and that provides easy installation for most Linux distributions. If you want to build and install DB-All.e yourself, you'll need to install the automake/autoconf/libtool packages then you can proceed as in most other Unix software: autoreconf -if ./configure make make install Getting started --------------- DB-All.e requires a database to run. It can create a SQLite database, or access a PostgreSQL or MySQL database. See doc/fapi_connect.md for details about connecting to a database. Once this is set up, you can initialise the DB-All.e database using the command:: dbadb wipe --url=sqlite:dballe.sqlite3 If you do not already have access to datasets to import, some are available from http://www.ncar.ucar.edu/tools/datasets/ after registering (for free) on the website. Documentation ------------- Documentation for all commandline tools can be found in their manpages. All commandline tools also have extensive commandline help that can be accessed using the "--help" option. The Fortran API is documented in the fapi.pdf document. The C API and all the C internals are documented through Doxygen. Administration and maintanance of DB-All.e are covered in the guide.pdf document. Testing DB-All.e ---------------- Unit testing can be run using "make check", but it requires an existing DSN connection to a MySQL database, which should be called 'test'. Please note that unit testing functions will wipe existing DB-All.e tables on the test DSN database. Useful resources ---------------- BUFR decoding: * ECWMF BUFR template codes: * Contact and copyright information --------------------------------- The author of DB-ALLe is Enrico Zini DB-ALLe is Copyright (C) 2005-2018 ARPAE-SIMC DB-ALLe is licensed under the terms of the GNU General Public License version 2. Please see the file COPYING for details. Contact informations for ARPAE-SIMC: Agenzia Regionale per la Prevenzione, l'Ambiente e l'Energia (ARPAE) Servizio Idro-Meteo-Climatologico (SIMC) Address: Viale Silvani 6, 40122 Bologna, Italy Tel: + 39 051 6497511 Fax: + 39 051 6497501 Email: urpsim@arpae.it Website: http://www.arpae.it/sim/ dballe-8.6/config.h.in0000644000175000017500000000455713602151747011577 00000000000000/* config.h.in. Generated from configure.ac by autoheader. */ /* define if the compiler supports basic C++11 syntax */ #undef HAVE_CXX11 /* Have Fortran bindings */ #undef HAVE_DBALLEF /* Have Python bindings */ #undef HAVE_DBALLE_PYTHON /* Define to 1 if you have the header file. */ #undef HAVE_DLFCN_H /* Define to 1 if you have the header file. */ #undef HAVE_INTTYPES_H /* PostgreSQL is available */ #undef HAVE_LIBPQ /* Define to 1 if you have the header file. */ #undef HAVE_MEMORY_H /* MySQL is available */ #undef HAVE_MYSQL /* popt.h has been found */ #undef HAVE_POPT_H /* Define to 1 if you have the header file. */ #undef HAVE_STDINT_H /* Define to 1 if you have the header file. */ #undef HAVE_STDLIB_H /* we can use strcasestr */ #undef HAVE_STRCASESTR /* Define to 1 if you have the header file. */ #undef HAVE_STRINGS_H /* Define to 1 if you have the header file. */ #undef HAVE_STRING_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_STAT_H /* Define to 1 if you have the header file. */ #undef HAVE_SYS_TYPES_H /* Define to 1 if you have the header file. */ #undef HAVE_UNISTD_H /* Define to the sub-directory where libtool stores uninstalled libraries. */ #undef LT_OBJDIR /* Name of package */ #undef PACKAGE /* Define to the address where bug reports for this package should be sent. */ #undef PACKAGE_BUGREPORT /* Define to the full name of this package. */ #undef PACKAGE_NAME /* Define to the full name and version of this package. */ #undef PACKAGE_STRING /* Define to the one symbol short name of this package. */ #undef PACKAGE_TARNAME /* Define to the home page for this package. */ #undef PACKAGE_URL /* Define to the version of this package. */ #undef PACKAGE_VERSION /* Define to 1 if you have the ANSI C header files. */ #undef STDC_HEADERS /* we need to use our own bswap_* functions */ #undef USE_OWN_BSWAP /* we need to use our own vasprintf */ #undef USE_OWN_VASPRINTF /* Version number of package */ #undef VERSION /* Enable large inode numbers on Mac OS X 10.5. */ #ifndef _DARWIN_USE_64_BIT_INODE # define _DARWIN_USE_64_BIT_INODE 1 #endif /* Number of bits in a file offset, on hosts where this is settable. */ #undef _FILE_OFFSET_BITS /* Define for large files, on AIX-style hosts. */ #undef _LARGE_FILES dballe-8.6/run-check0000755000175000017500000000060213554564112011344 00000000000000#!/bin/sh TOP_SRCDIR=$(cd $(dirname $0) && pwd) export LD_LIBRARY_PATH="$TOP_SRCDIR/dballe:$LD_LIBRARY_PATH" export WREPORT_EXTRA_TABLES=$TOP_SRCDIR/tables for conffile in ./run-check.conf .git/run-check.conf do if [ -f "$conffile" ] then . "$conffile" export DBA_DB export DBA_DB_SQLITE export DBA_DB_POSTGRESQL export DBA_DB_MYSQL break fi done exec make check "$@" dballe-8.6/Makefile.am0000644000175000017500000000263013602147062011571 00000000000000## Process this file with automake to produce Makefile.in ACLOCAL_AMFLAGS = -I m4 SUBDIRS = tables dballe extra src if DO_DBALLEF SUBDIRS += fortran endif if DO_DBALLE_PYTHON SUBDIRS += python endif if DO_DOCS SUBDIRS += doc endif SUBDIRS += bench . pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA=libdballe.pc libdballef.pc m4dir = $(datadir)/aclocal m4_DATA = libdballe.m4 libdballef.m4 man_MANS = dbatbl.1 dbamsg.1 dbadb.1 dbaexport.1 dbadb.1: src/dbadb doc/add_templates_to_manpage ( cd src && ./`basename $<` help manpage ) | ( cd $(srcdir) && $(srcdir)/doc/add_templates_to_manpage ) > $@ dbamsg.1: src/dbamsg doc/add_templates_to_manpage ( cd src && ./`basename $<` help manpage ) | ( cd $(srcdir) && $(srcdir)/doc/add_templates_to_manpage ) > $@ dbaexport.1: src/dbaexport COLUMNS=200 help2man --name='export data from DB-All.e' --section=1 --no-info --version-string="$(PACKAGE_VERSION)" $< > $@ %.1: src/% ( cd src && ./`basename $<` help manpage ) > $@ BUILT_SOURCES = dbaexport.1 EXTRA_DIST = README.md NEWS.md LICENSE BENCHMARKS autogen.sh \ libdballe.m4 libdballef.m4 \ libdballe.pc.in libdballef.pc.in \ $(conf_DATA) get_line_no run-check.conf.example \ run-check run-local run-bench show_code_notes \ fabfile.py \ UPGRADE-5.x \ dbaexport.1 \ rebuild-gh-pages \ fedora/SPECS/dballe.spec CLEANFILES = dbatbl.1 dbamsg.1 dbadb.1 fabfile.pyc dballe-8.6/dballe/0000755000175000017500000000000013602152021011026 500000000000000dballe-8.6/dballe/values.cc0000644000175000017500000002127313554564112012577 00000000000000#include "values.h" #include "core/values.h" #include using namespace std; using namespace wreport; namespace dballe { namespace impl { template bool ValuesBase::operator==(const ValuesBase& o) const { return m_values == o.m_values; } template bool ValuesBase::operator!=(const ValuesBase& o) const { return m_values != o.m_values; } template typename ValuesBase::iterator ValuesBase::find(wreport::Varcode code) noexcept { /* Binary search */ if (m_values.empty()) return m_values.end(); iterator low = m_values.begin(), high = (m_values.end() - 1); while (low <= high) { iterator middle = low + (high - low) / 2; int cmp = (int)code - (int)(middle->code()); if (cmp < 0) high = middle - 1; else if (cmp > 0) low = middle + 1; else return middle; } return m_values.end(); } template typename ValuesBase::const_iterator ValuesBase::find(wreport::Varcode code) const noexcept { /* Binary search */ if (m_values.empty()) return m_values.end(); const_iterator low = m_values.cbegin(), high = (m_values.cend() - 1); while (low <= high) { const_iterator middle = low + (high - low) / 2; int cmp = (int)code - (int)middle->code(); if (cmp < 0) high = middle - 1; else if (cmp > 0) low = middle + 1; else return middle; } return m_values.end(); } template typename ValuesBase::iterator ValuesBase::insert_new(Value&& val) { // Insertionsort, since the common case is to work with small arrays wreport::Varcode key = val.code(); // Enlarge the buffer m_values.resize(m_values.size() + 1); // Insertionsort iterator pos; for (pos = m_values.end() - 1; pos > m_values.begin(); --pos) { if ((pos - 1)->code() > key) *pos = std::move(*(pos - 1)); else break; } *pos = std::move(val); return pos; } template void ValuesBase::unset(wreport::Varcode code) { iterator pos = find(code); if (pos == end()) return; m_values.erase(pos); } template void ValuesBase::set(Value&& val) { auto i = find(val.code()); if (i == end()) insert_new(std::move(val)); else *i = std::move(val); } template void ValuesBase::set(const wreport::Var& v) { auto i = find(v.code()); if (i == end()) insert_new(Value(v)); else i->reset(v); } template void ValuesBase::set(std::unique_ptr&& v) { auto code = v->code(); auto i = find(code); if (i == end()) insert_new(Value(move(v))); else i->reset(std::move(v)); } template void ValuesBase::merge(const ValuesBase& vals) { for (const auto& vi: vals) set(*vi); } template void ValuesBase::merge(ValuesBase&& vals) { if (empty()) operator=(std::move(vals)); else { for (const auto& vi: vals) set(std::move(*vi)); vals.clear(); } } template const Value& ValuesBase::value(wreport::Varcode code) const { auto i = find(code); if (i == end()) error_notfound::throwf("variable %01d%02d%03d not found", WR_VAR_F(code), WR_VAR_X(code), WR_VAR_Y(code)); return *i; } template const wreport::Var& ValuesBase::var(wreport::Varcode code) const { auto i = find(code); if (i == end()) error_notfound::throwf("variable %01d%02d%03d not found", WR_VAR_F(code), WR_VAR_X(code), WR_VAR_Y(code)); if (!i->get()) error_notfound::throwf("variable %01d%02d%03d not set", WR_VAR_F(code), WR_VAR_X(code), WR_VAR_Y(code)); return **i; } template wreport::Var& ValuesBase::var(wreport::Varcode code) { auto i = find(code); if (i == end()) error_notfound::throwf("variable %01d%02d%03d not found", WR_VAR_F(code), WR_VAR_X(code), WR_VAR_Y(code)); if (!i->get()) error_notfound::throwf("variable %01d%02d%03d not set", WR_VAR_F(code), WR_VAR_X(code), WR_VAR_Y(code)); return **i; } template const Value* ValuesBase::maybe_value(wreport::Varcode code) const { auto i = find(code); if (i == end()) return nullptr; return &*i; } template const wreport::Var* ValuesBase::maybe_var(wreport::Varcode code) const { auto i = find(code); if (i == end()) return nullptr; if (!i->get()) return nullptr; return i->get(); } template wreport::Var* ValuesBase::maybe_var(wreport::Varcode code) { auto i = find(code); if (i == end()) return nullptr; if (!i->get()) return nullptr; return i->get(); } template void ValuesBase::move_to(std::function)> dest) { for (auto& val: m_values) dest(val.release()); m_values.clear(); } template void ValuesBase::move_to_attributes(wreport::Var& dest) { for (auto& val: m_values) dest.seta(val.release()); m_values.clear(); } template void ValuesBase::print(FILE* out) const { for (const auto& val: *this) val.print(out); } template std::vector ValuesBase::encode() const { core::value::Encoder enc; for (const auto& i: *this) enc.append(*i); return enc.buf; } template std::vector ValuesBase::encode_attrs(const wreport::Var& var) { core::value::Encoder enc; for (const Var* a = var.next_attr(); a != nullptr; a = a->next_attr()) enc.append(*a); return enc.buf; } template void ValuesBase::decode(const std::vector& buf, std::function)> dest) { core::value::Decoder dec(buf); while (dec.size) dest(move(dec.decode_var())); } template struct ValuesBase; template struct ValuesBase; } Values::Values(const DBValues& o) { clear(); reserve(o.size()); for (const auto& val: o) if (const Var* var = val.get()) m_values.emplace_back(*var); } Values::Values(DBValues&& o) { clear(); reserve(o.size()); o.move_to([&](std::unique_ptr var) { m_values.emplace_back(std::move(var)); }); } Values& Values::operator=(const DBValues& o) { clear(); reserve(o.size()); for (const auto& val: o) if (const Var* var = val.get()) m_values.emplace_back(*var); return *this; } Values& Values::operator=(DBValues&& o) { clear(); reserve(o.size()); o.move_to([&](std::unique_ptr var) { m_values.emplace_back(std::move(var)); }); return *this; } DBValues::DBValues(const Values& o) { clear(); reserve(o.size()); for (const auto& val: o) if (const Var* var = val.get()) m_values.emplace_back(*var); } DBValues::DBValues(Values&& o) { clear(); reserve(o.size()); o.move_to([&](std::unique_ptr var) { m_values.emplace_back(std::move(var)); }); } DBValues& DBValues::operator=(const Values& o) { clear(); reserve(o.size()); for (const auto& val: o) if (const Var* var = val.get()) m_values.emplace_back(*var); return *this; } DBValues& DBValues::operator=(Values&& o) { clear(); reserve(o.size()); o.move_to([&](std::unique_ptr var) { m_values.emplace_back(std::move(var)); }); return *this; } bool DBValues::vars_equal(const DBValues& o) const { const_iterator a = begin(); const_iterator b = o.begin(); while (a != end() && b != o.end()) { if (!a->get() && !b->get()) ; else if (!a->get() || !b->get()) return false; else if (*a->get() != *b->get()) return false; ++a; ++b; } return a == end() && b == o.end(); } void DBValues::set_data_id(wreport::Varcode code, int data_id) { auto i = find(code); if (i == end()) return; i->data_id = data_id; } std::ostream& operator<<(std::ostream& o, const Values& values) { for (const auto& val: values) o << val << endl; return o; } std::ostream& operator<<(std::ostream& o, const DBValues& values) { for (const auto& val: values) o << val << endl; return o; } } dballe-8.6/dballe/file.cc0000644000175000017500000000632613554564112012221 00000000000000#include "file.h" #include "core/file.h" #include #include #include #include #include #include using namespace std; using namespace wreport; namespace dballe { BinaryMessage::operator bool() const { return !data.empty(); } File::~File() { } const char* File::encoding_name(Encoding enc) { switch (enc) { case Encoding::BUFR: return "BUFR"; case Encoding::CREX: return "CREX"; default: error_notfound::throwf("unsupported encoding value %d", static_cast(enc)); } } Encoding File::parse_encoding(const char* s) { std::string str(s); return parse_encoding(str); } Encoding File::parse_encoding(const std::string& s) { std::string str = wreport::str::upper(s); if (str == "BUFR") return Encoding::BUFR; if (str == "CREX") return Encoding::CREX; error_notfound::throwf("unsupported encoding '%s'", s.c_str()); } unique_ptr File::create(const std::string& pathname, const char* mode) { FILE* fp = fopen(pathname.c_str(), mode); if (fp == NULL) error_system::throwf("opening %s with mode '%s'", pathname.c_str(), mode); return File::create(fp, true, pathname); } unique_ptr File::create(Encoding type, const std::string& pathname, const char* mode) { FILE* fp = fopen(pathname.c_str(), mode); if (fp == NULL) error_system::throwf("opening %s with mode '%s'", pathname.c_str(), mode); return File::create(type, fp, true, pathname); } namespace { struct stream_tracker { FILE* stream; bool close_on_exit; stream_tracker(FILE* stream, bool close_on_exit) : stream(stream), close_on_exit(close_on_exit) {} ~stream_tracker() { if (stream && close_on_exit) fclose(stream); } FILE* release() { FILE* res = stream; stream = nullptr; return res; } }; } unique_ptr File::create(FILE* stream, bool close_on_exit, const std::string& name) { stream_tracker st(stream, close_on_exit); // Auto-detect from the first character in the stream int c = getc(stream); // In case of EOF, pick any type that will handle EOF gracefully. if (c == EOF) return create(Encoding::BUFR, st.release(), close_on_exit, name); if (ungetc(c, stream) == EOF) error_system::throwf("cannot put the first byte of %s back into the input stream", name.c_str()); switch (c) { case 'B': return create(Encoding::BUFR, st.release(), close_on_exit, name); case 'C': return create(Encoding::CREX, st.release(), close_on_exit, name); default: throw error_notfound("could not detect the encoding of " + name); } } unique_ptr File::create(Encoding type, FILE* stream, bool close_on_exit, const std::string& name) { switch (type) { case Encoding::BUFR: return unique_ptr(new core::BufrFile(name, stream, close_on_exit)); case Encoding::CREX: return unique_ptr(new core::CrexFile(name, stream, close_on_exit)); default: error_consistency::throwf("cannot handle unknown file type %d", (int)type); } } std::ostream& operator<<(std::ostream& o, const dballe::Encoding& e) { return o << File::encoding_name(e); } } dballe-8.6/dballe/query.cc0000644000175000017500000000026313554564112012441 00000000000000#include "query.h" #include "core/query.h" using namespace std; namespace dballe { std::unique_ptr Query::create() { return unique_ptr(new core::Query); } } dballe-8.6/dballe/types.cc0000644000175000017500000013263213572457034012452 00000000000000#include "types.h" #include "core/csv.h" #include #include #include #include #include using namespace wreport; using namespace std; namespace dballe { /* * Date */ Date::Date() : year(0xffff), month(0xff), day(0xff) { } Date::Date(int ye, int mo, int da) { if (ye == MISSING_INT) { year = 0xffff; month = day = 0xff; } else { Date::validate(ye, mo, da); year = ye; month = mo; day = da; } } void Date::validate(int ye, int mo, int da) { if (mo == MISSING_INT || mo < 1 || mo > 12) error_consistency::throwf("month %d is not between 1 and 12", mo); if (da == MISSING_INT || da < 1 || da > days_in_month(ye, mo)) error_consistency::throwf("day %d is not between 1 and %d", da, days_in_month(ye, mo)); } bool Date::is_missing() const { return year == 0xffff; } int Date::compare(const Date& o) const { if (int res = year - o.year) return res; if (int res = month - o.month) return res; return day - o.day; } int Date::to_julian() const { return calendar_to_julian(year, month, day); } Date Date::from_julian(int jday) { Date res; julian_to_calendar(jday, res.year, res.month, res.day); return res; } int Date::calendar_to_julian(int year, int month, int day) { // From http://libpqtypes.esilo.com/browse_source.html?file=datetime.c if (month > 2) { month += 1; year += 4800; } else { month += 13; year += 4799; } int century = year / 100; int julian = year * 365 - 32167; julian += year / 4 - century + century / 4; julian += 7834 * month / 256 + day; return julian; } void Date::julian_to_calendar(int jday, unsigned short& year, unsigned char& month, unsigned char& day) { // From http://libpqtypes.esilo.com/browse_source.html?file=datetime.c unsigned julian = jday + 32044; unsigned quad = julian / 146097; unsigned extra = (julian - quad * 146097) * 4 + 3; julian += 60 + quad * 3 + extra / 146097; quad = julian / 1461; julian -= quad * 1461; int y = julian * 4 / 1461; julian = ((y != 0) ? ((julian + 305) % 365) : ((julian + 306) % 366)) + 123; y += quad * 4; year = y - 4800; quad = julian * 2141 / 65536; day = julian - 7834 * quad / 256; month = (quad + 10) % 12 + 1; } bool Date::operator==(const Date& dt) const { return year == dt.year && month == dt.month && day == dt.day; } bool Date::operator!=(const Date& dt) const { return year != dt.year || month != dt.month || day != dt.day; } bool Date::operator<(const Date& dt) const { if (year < dt.year) return true; if (year > dt.year) return false; if (month < dt.month) return true; if (month > dt.month) return false; return day < dt.day; } bool Date::operator>(const Date& dt) const { if (year < dt.year) return false; if (year > dt.year) return true; if (month < dt.month) return false; if (month > dt.month) return true; return day > dt.day; } int Date::days_in_month(int year, int month) { switch (month) { case 1: return 31; case 2: if (year % 400 == 0 || (year % 4 == 0 && ! (year % 100 == 0))) return 29; return 28; case 3: return 31; case 4: return 30; case 5: return 31; case 6: return 30; case 7: return 31; case 8: return 31; case 9: return 30; case 10: return 31; case 11: return 30; case 12: return 31; default: error_consistency::throwf("Month %d is not between 1 and 12", month); } } void Date::to_stream_iso8601(std::ostream& out) const { out << setw(4) << setfill('0') << year << '-' << setw(2) << setfill('0') << (unsigned)month << '-' << setw(2) << setfill('0') << (unsigned)day; } void Date::to_csv_iso8601(CSVWriter& out) const { if (is_missing()) out.add_value_empty(); else { char buf[16]; snprintf(buf, 16, "%04hu-%02hhu-%02hhu", year, month, day); out.add_value(buf); } } std::ostream& operator<<(std::ostream& out, const Date& dt) { dt.to_stream_iso8601(out); return out; } /* * Time */ Time::Time() : hour(0xff), minute(0xff), second(0xff) { } Time::Time(int ho, int mi, int se) { Time::validate(ho, mi, se); hour = ho; minute = mi; second = se; } void Time::validate(int ho, int mi, int se) { if (ho == MISSING_INT || ho < 0 || ho > 23) error_consistency::throwf("hour %d is not between 1 and 23", ho); if (mi == MISSING_INT || mi < 0 || mi > 59) error_consistency::throwf("minute %d is not between 1 and 59", mi); if (ho == 23 && mi == 59) { if (se == MISSING_INT || se < 0 || se > 60) error_consistency::throwf("second %d is not between 1 and 60", se); } else { if (se == MISSING_INT || se < 0 || se > 59) error_consistency::throwf("second %d is not between 1 and 59", se); } } bool Time::is_missing() const { return hour == 0xff; } int Time::compare(const Time& o) const { if (int res = hour - o.hour) return res; if (int res = minute - o.minute) return res; return second - o.second; } bool Time::operator==(const Time& dt) const { return hour == dt.hour && minute == dt.minute && second == dt.second; } bool Time::operator!=(const Time& dt) const { return hour != dt.hour || minute != dt.minute || second != dt.second; } bool Time::operator<(const Time& dt) const { if (hour < dt.hour) return true; if (hour > dt.hour) return false; if (minute < dt.minute) return true; if (minute > dt.minute) return false; return second < dt.second; } bool Time::operator>(const Time& dt) const { if (hour < dt.hour) return false; if (hour > dt.hour) return true; if (minute < dt.minute) return false; if (minute > dt.minute) return true; return second > dt.second; } void Time::to_stream_iso8601(std::ostream& out) const { out << setw(2) << setfill('0') << (unsigned)hour << ':' << setw(2) << setfill('0') << (unsigned)minute << ':' << setw(2) << setfill('0') << (unsigned)second; } void Time::to_csv_iso8601(CSVWriter& out) const { if (is_missing()) out.add_value_empty(); else { char buf[16]; snprintf(buf, 16, "%02hhu:%02hhu:%02hhu", hour, minute, second); out.add_value(buf); } } std::ostream& operator<<(std::ostream& out, const Time& dt) { dt.to_stream_iso8601(out); return out; } /* * Datetime */ Datetime::Datetime() : year(0xffff), month(0xff), day(0xff), hour(0xff), minute(0xff), second(0xff) { } Datetime::Datetime(const Date& date, const Time& time) : year(date.year), month(date.month), day(date.day), hour(time.hour), minute(time.minute), second(time.second) {} Datetime::Datetime(int ye, int mo, int da, int ho, int mi, int se) { if (ye == MISSING_INT) { year = 0xffff; month = day = hour = minute = second = 0xff; } else { Datetime::validate(ye, mo, da, ho, mi, se); year = ye; month = mo; day = da; hour = ho; minute = mi; second = se; } } void Datetime::validate(int ye, int mo, int da, int ho, int mi, int se) { Date::validate(ye, mo, da); Time::validate(ho, mi, se); } void Datetime::normalise_h24(int& ye, int& mo, int& da, int& ho, int& mi, int& se) { if (ho != 24 || mi != 0 || se != 0) return; ho = 0; ++da; if (da > Date::days_in_month(ye, mo)) { da = 1; ++mo; if (mo > 12) { mo = 1; ++ye; } } } Datetime Datetime::from_julian(int jday, int ho, int mi, int se) { return Datetime(Date::from_julian(jday), Time(ho, mi, se)); } static void check_partial_consistency(const Datetime& dt) { if (dt.year == 0xffff) { if (dt.month != 0xff) error_consistency::throwf("month %d given with no year", dt.month); if (dt.day != 0xff) error_consistency::throwf("day %d given with no year", dt.day); if (dt.hour != 0xff) error_consistency::throwf("hour %d given with no year", dt.hour); if (dt.minute != 0xff) error_consistency::throwf("minute %d given with no year", dt.minute); if (dt.second != 0xff) error_consistency::throwf("second %d given with no year", dt.second); } if (dt.month == 0xff) { if (dt.day != 0xff) error_consistency::throwf("day %d given with no month", dt.day); } if (dt.hour == 0xff) { if (dt.minute != 0xff) error_consistency::throwf("minute %d given with no hour", dt.minute); if (dt.second != 0xff) error_consistency::throwf("second %d given with no hour", dt.second); } if (dt.minute == 0xff) { if (dt.second != 0xff) error_consistency::throwf("second %d given with no minute", dt.second); } } static void check_partial_consistency(int ye, int mo, int da, int ho, int mi, int se) { if (ye == MISSING_INT) { if (mo != MISSING_INT) error_consistency::throwf("month %d given with no year", mo); if (da != MISSING_INT) error_consistency::throwf("day %d given with no year", da); if (ho != MISSING_INT) error_consistency::throwf("hour %d given with no year", ho); if (mi != MISSING_INT) error_consistency::throwf("minute %d given with no year", mi); if (se != MISSING_INT) error_consistency::throwf("second %d given with no year", se); } if (mo == MISSING_INT) { if (da != MISSING_INT) error_consistency::throwf("day %d given with no month", da); } if (ho == MISSING_INT) { if (mi != MISSING_INT) error_consistency::throwf("minute %d given with no hour", mi); if (se != MISSING_INT) error_consistency::throwf("second %d given with no hour", se); } if (mi == MISSING_INT) { if (se != MISSING_INT) error_consistency::throwf("second %d given with no minute", se); } } Datetime Datetime::lower_bound(int ye, int mo, int da, int ho, int mi, int se) { check_partial_consistency(ye, mo, da, ho, mi, se); if (ye == MISSING_INT) return Datetime(); if (mo == MISSING_INT) mo = 1; if (da == MISSING_INT) da = 1; if (ho == MISSING_INT) ho = 0; if (mi == MISSING_INT) mi = 0; if (se == MISSING_INT) se = 0; return Datetime(ye, mo, da, ho, mi, se); } Datetime Datetime::upper_bound(int ye, int mo, int da, int ho, int mi, int se) { check_partial_consistency(ye, mo, da, ho, mi, se); if (ye == MISSING_INT) return Datetime(); if (mo == MISSING_INT) mo = 12; if (da == MISSING_INT) da = Date::days_in_month(ye, mo); if (ho == MISSING_INT) ho = 23; if (mi == MISSING_INT) mi = 59; if (se == MISSING_INT) se = 59; return Datetime(ye, mo, da, ho, mi, se); } void Datetime::set_lower_bound() { check_partial_consistency(*this); if (year == 0xffff) { month = day = hour = minute = second = 0xff; return; } if (month == 0xff) month = 1; if (day == 0xff) day = 1; if (hour == 0xff) hour = 0; if (minute == 0xff) minute = 0; if (second == 0xff) second = 0; } void Datetime::set_upper_bound() { check_partial_consistency(*this); if (year == 0xffff) { month = day = hour = minute = second = 0xff; return; } if (month == 0xff) month = 12; if (day == 0xff) day = Date::days_in_month(year, month); if (hour == 0xff) hour = 23; if (minute == 0xff) minute = 59; if (second == 0xff) second = 59; } Date Datetime::date() const { return is_missing() ? Date() : Date(year, month, day); } Time Datetime::time() const { return is_missing() ? Time() : Time(hour, minute, second); } bool Datetime::is_missing() const { return year == 0xffff; } int Datetime::to_julian() const { return Date::calendar_to_julian(year, month, day); } int Datetime::compare(const Datetime& o) const { if (int res = year - o.year) return res; if (int res = month - o.month) return res; if (int res = day - o.day) return res; if (int res = hour - o.hour) return res; if (int res = minute - o.minute) return res; return second - o.second; } bool Datetime::operator==(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) == std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } bool Datetime::operator!=(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) != std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } bool Datetime::operator<(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) < std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } bool Datetime::operator>(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) > std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } bool Datetime::operator<=(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) <= std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } bool Datetime::operator>=(const Datetime& o) const { return std::tie(year, month, day, hour, minute, second) >= std::tie(o.year, o.month, o.day, o.hour, o.minute, o.second); } Datetime Datetime::from_iso8601(const char* str) { int ye, mo, da, ho, mi, se; char sep; if (sscanf(str, "%04d-%02d-%02d%c%02d:%02d:%02d", &ye, &mo, &da, &sep, &ho, &mi, &se) != 7) error_consistency::throwf("cannot parse date/time string \"%s\"", str); if (sep != 'T' && sep != ' ') error_consistency::throwf("invalid iso8601 separator '%c' in datetime string \"%s\"", sep, str); return Datetime(ye, mo, da, ho, mi, se); } void Datetime::to_stream_iso8601(std::ostream& out, char sep, const char* tz) const { out << setw(4) << setfill('0') << year << '-' << setw(2) << setfill('0') << (unsigned)month << '-' << setw(2) << setfill('0') << (unsigned)day << sep << setw(2) << setfill('0') << (unsigned)hour << ':' << setw(2) << setfill('0') << (unsigned)minute << ':' << setw(2) << setfill('0') << (unsigned)second << tz; } int Datetime::print_iso8601(FILE* out, char sep, const char* end) const { return fprintf(out, "%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu%s", year, month, day, sep, hour, minute, second, end); } int Datetime::print(FILE* out, const char* end) const { return print_iso8601(out, 'T', end); } void Datetime::to_csv_iso8601(CSVWriter& out, char sep, const char* tz) const { if (is_missing()) out.add_value_empty(); else { char buf[32]; snprintf(buf, 32, "%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu%s", year, month, day, sep, hour, minute, second, tz); out.add_value(buf); } } std::string Datetime::to_string(char sep, const char* tz) const { char buf[32]; int len = snprintf(buf, 32, "%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu%s", year, month, day, sep, hour, minute, second, tz); return std::string(buf, len); } std::ostream& operator<<(std::ostream& out, const Datetime& dt) { dt.to_stream_iso8601(out); return out; } /* * DatetimeRange */ DatetimeRange::DatetimeRange( int yemin, int momin, int damin, int homin, int mimin, int semin, int yemax, int momax, int damax, int homax, int mimax, int semax) { set(yemin, momin, damin, homin, mimin, semin, yemax, momax, damax, homax, mimax, semax); } bool DatetimeRange::is_missing() const { return min.is_missing() && max.is_missing(); } bool DatetimeRange::operator==(const DatetimeRange& o) const { return std::tie(min, max) == std::tie(o.min, o.max); } bool DatetimeRange::operator!=(const DatetimeRange& o) const { return std::tie(min, max) != std::tie(o.min, o.max); } bool DatetimeRange::operator<(const DatetimeRange& o) const { return std::tie(min, max) < std::tie(o.min, o.max); } bool DatetimeRange::operator<=(const DatetimeRange& o) const { return std::tie(min, max) <= std::tie(o.min, o.max); } bool DatetimeRange::operator>(const DatetimeRange& o) const { return std::tie(min, max) > std::tie(o.min, o.max); } bool DatetimeRange::operator>=(const DatetimeRange& o) const { return std::tie(min, max) >= std::tie(o.min, o.max); } void DatetimeRange::merge(const DatetimeRange& range) { if (!min.is_missing() && (range.min.is_missing() || range.min < min)) min = range.min; if (!max.is_missing() && (range.max.is_missing() || range.max > max)) max = range.max; } void DatetimeRange::set(const Datetime& min, const Datetime& max) { this->min = min; this->max = max; } void DatetimeRange::set( int yemin, int momin, int damin, int homin, int mimin, int semin, int yemax, int momax, int damax, int homax, int mimax, int semax) { min = Datetime::lower_bound(yemin, momin, damin, homin, mimin, semin); max = Datetime::upper_bound(yemax, momax, damax, homax, mimax, semax); } bool DatetimeRange::contains(const Datetime& dt) const { if (min.is_missing()) if (max.is_missing()) return true; else return dt <= max; else if (max.is_missing()) return dt >= min; else return min <= dt && dt <= max; } bool DatetimeRange::contains(const DatetimeRange& dtr) const { if (!min.is_missing() && (dtr.min.is_missing() || dtr.min < min)) return false; if (!max.is_missing() && (dtr.max.is_missing() || dtr.max > max)) return false; return true; } bool DatetimeRange::is_disjoint(const DatetimeRange& dtr) const { if (!max.is_missing() && !dtr.min.is_missing() && max < dtr.min) return true; if (!dtr.max.is_missing() && !min.is_missing() && dtr.max < min) return true; return false; } int DatetimeRange::print(FILE* out, const char* end) const { int res = min.print(out, " to "); res += max.print(out, end); return res; } std::ostream& operator<<(std::ostream& out, const DatetimeRange& dtr) { if (dtr.min == dtr.max) dtr.min.to_stream_iso8601(out); else { out << "("; dtr.min.to_stream_iso8601(out); out << " to "; dtr.max.to_stream_iso8601(out); out << ")"; } return out; } /* * Coordinate utilities */ namespace { inline int ll_to_int(double ll) { return lround(ll * 100000.0); } inline double ll_from_int(int ll) { return (double)ll / 100000.0; } inline int normalon(int lon) { return ((lon + 18000000) % 36000000) - 18000000; } } /* * Coords */ Coords::Coords(int lat, int lon) { set(lat, lon); } Coords::Coords(double lat, double lon) : lat(ll_to_int(lat)), lon(normalon(ll_to_int(lon))) { } bool Coords::is_missing() const { return lat == MISSING_INT && lon == MISSING_INT; } void Coords::set_lat(double lat) { this->lat = ll_to_int(lat); } void Coords::set_lon(double lon) { this->lon = normalon(ll_to_int(lon)); } void Coords::set_lat(int lat) { if (lat == MISSING_INT) this->lat = this->lon = MISSING_INT; else this->lat = lat; } void Coords::set_lon(int lon) { if (lon == MISSING_INT) this->lat = this->lon = MISSING_INT; else this->lon = normalon(lon); } void Coords::set(int lat, int lon) { if (lat == MISSING_INT || lon == MISSING_INT) this->lat = this->lon = MISSING_INT; else { this->lat = lat; this->lon = normalon(lon); } } void Coords::set(double lat, double lon) { this->lat = ll_to_int(lat); this->lon = normalon(ll_to_int(lon)); } double Coords::dlat() const { return ll_from_int(lat); } double Coords::dlon() const { return ll_from_int(lon); } int Coords::compare(const Coords& o) const { if (int res = lat - o.lat) return res; return lon - o.lon; } bool Coords::operator==(const Coords& o) const { return std::tie(lat, lon) == std::tie(o.lat, o.lon); } bool Coords::operator!=(const Coords& o) const { return std::tie(lat, lon) != std::tie(o.lat, o.lon); } bool Coords::operator<(const Coords& o) const { return std::tie(lat, lon) < std::tie(o.lat, o.lon); } bool Coords::operator>(const Coords& o) const { return std::tie(lat, lon) > std::tie(o.lat, o.lon); } bool Coords::operator<=(const Coords& o) const { return std::tie(lat, lon) <= std::tie(o.lat, o.lon); } bool Coords::operator>=(const Coords& o) const { return std::tie(lat, lon) >= std::tie(o.lat, o.lon); } int Coords::print(FILE* out, const char* end) const { return fprintf(out, "%.5f,%.5f%s", dlat(), dlon(), end); } std::string Coords::to_string(const char* undef) const { string res; if (lat == MISSING_INT) res += undef; else res += std::to_string(dlat()); res += ","; if (lon == MISSING_INT) res += undef; else res += std::to_string(dlon()); return res; } std::ostream& operator<<(std::ostream& out, const Coords& c) { if (c.is_missing()) return out << "(-,-)"; else return out << fixed << "(" << setprecision(5) << c.dlat() << "," << setprecision(5) << c.dlon() << ")" // << resetiosflags(ios_base::floatfield); // << defaultfloat; ; } int Coords::lat_to_int(double lat) { return ll_to_int(lat); } int Coords::lon_to_int(double lon) { return normalon(ll_to_int(lon)); } double Coords::lat_from_int(int lat) { return ll_from_int(lat); } double Coords::lon_from_int(int lon) { return ll_from_int(lon); } /* * LatRange */ constexpr int LatRange::IMIN; constexpr int LatRange::IMAX; constexpr double LatRange::DMIN; constexpr double LatRange::DMAX; LatRange::LatRange(int min, int max) : imin(min == MISSING_INT ? LatRange::IMIN : min), imax(max == MISSING_INT ? LatRange::IMAX : max) {} LatRange::LatRange(double min, double max) : imin(ll_to_int(min)), imax(ll_to_int(max)) { } bool LatRange::operator==(const LatRange& lr) const { return imin == lr.imin && imax == lr.imax; } bool LatRange::operator!=(const LatRange& lr) const { return imin != lr.imin || imax != lr.imax; } bool LatRange::is_missing() const { return imin == IMIN && imax == IMAX; } double LatRange::dmin() const { return ll_from_int(imin); } double LatRange::dmax() const { return ll_from_int(imax); } void LatRange::get(double& min, double& max) const { min = ll_from_int(imin); max = ll_from_int(imax); } void LatRange::set(int min, int max) { imin = min == MISSING_INT ? LatRange::IMIN : min; imax = max == MISSING_INT ? LatRange::IMAX : max; } void LatRange::set(double min, double max) { imin = ll_to_int(min); imax = ll_to_int(max); } bool LatRange::contains(int lat) const { return lat >= imin && lat <= imax; } bool LatRange::contains(double lat) const { int ilat = ll_to_int(lat); return ilat >= imin && ilat <= imax; } bool LatRange::contains(const LatRange& lr) const { return imin <= lr.imin && lr.imax <= imax; } int LatRange::print(FILE* out, const char* end) const { double dmin, dmax; get(dmin, dmax); return fprintf(out, "(%.5f to %.5f)%s", dmin, dmax, end); } std::ostream& operator<<(std::ostream& out, const LatRange& lr) { double dmin, dmax; lr.get(dmin, dmax); out << fixed << "(" << setprecision(5) << dmin << " to " << setprecision(5) << dmax << ")" << resetiosflags(ios_base::floatfield); // << defaultfloat; return out; } /* * LonRange */ LonRange::LonRange(int min, int max) { set(min, max); } LonRange::LonRange(double min, double max) : imin(normalon(ll_to_int(min))), imax(normalon(ll_to_int(max))) { if (min != max && imin == imax) imin = imax = MISSING_INT; } bool LonRange::operator==(const LonRange& lr) const { if ((imin == MISSING_INT || imax == MISSING_INT) && (lr.imin == MISSING_INT || lr.imax == MISSING_INT)) return true; return imin == lr.imin && imax == lr.imax; } bool LonRange::operator!=(const LonRange& lr) const { if ((imin == MISSING_INT || imax == MISSING_INT) && (lr.imin == MISSING_INT || lr.imax == MISSING_INT)) return false; return imin != lr.imin || imax != lr.imax; } bool LonRange::is_missing() const { return imin == MISSING_INT || imax == MISSING_INT; } double LonRange::dmin() const { return imin == MISSING_INT ? -180.0 : ll_from_int(imin); } double LonRange::dmax() const { return imax == MISSING_INT ? 180.0 : ll_from_int(imax); } void LonRange::get(double& min, double& max) const { if (is_missing()) { min = -180.0; max = 180.0; } else { min = ll_from_int(imin); max = ll_from_int(imax); } } void LonRange::set(int min, int max) { if ((min != MISSING_INT || max != MISSING_INT) && (min == MISSING_INT || max == MISSING_INT)) error_consistency::throwf("cannot set longitude range to an open ended range"); imin = min == MISSING_INT ? MISSING_INT : normalon(min); imax = max == MISSING_INT ? MISSING_INT : normalon(max); // Catch cases like min=0 max=360, that would match anything, and set them // to missing range match if (min != max && imin == imax) imin = imax = MISSING_INT; } void LonRange::set(double min, double max) { set(ll_to_int(min), ll_to_int(max)); } void LonRange::set(const LonRange& lr) { set(lr.imin, lr.imax); } bool LonRange::contains(int lon) const { if (imin == imax) { if (imin == MISSING_INT) return true; return lon == imin; } else if (imin < imax) { return lon >= imin && lon <= imax; } else { return ((lon >= imin and lon <= 18000000) or (lon >= -18000000 and lon <= imax)); } } bool LonRange::contains(double lon) const { return contains(ll_to_int(lon)); } bool LonRange::contains(const LonRange& lr) const { if (is_missing()) return true; if (lr.is_missing()) return false; // Longitude ranges can match outside or inside the interval if (imin < imax) { // we match inside the interval if (lr.imin < lr.imax) { // lr matches inside the interval return imin <= lr.imin && lr.imax <= imax; } else { // lr matches outside the interval return false; } } else { // we match outside the interval if (lr.imin < lr.imax) { // lr matches inside the interval return lr.imax <= imin || lr.imin >= imax; } else { // lr matches outside the interval return lr.imin <= imin || lr.imax >= imax; } } } int LonRange::print(FILE* out, const char* end) const { double dmin, dmax; get(dmin, dmax); return fprintf(out, "(%.5f to %.5f)%s", dmin, dmax, end); } std::ostream& operator<<(std::ostream& out, const LonRange& lr) { double dmin, dmax; lr.get(dmin, dmax); out << fixed << "(" << setprecision(5) << dmin << " to " << setprecision(5) << dmax << ")" << resetiosflags(ios_base::floatfield); // << defaultfloat; return out; } /* * Level */ bool Level::is_missing() const { return ltype1 == MISSING_INT && l1 == MISSING_INT && ltype2 == MISSING_INT && l2 == MISSING_INT; } bool Level::operator==(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) == std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } bool Level::operator!=(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) != std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } bool Level::operator<(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) < std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } bool Level::operator>(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) > std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } bool Level::operator<=(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) <= std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } bool Level::operator>=(const Level& o) const { return std::tie(ltype1, l1, ltype2, l2) >= std::tie(o.ltype1, o.l1, o.ltype2, o.l2); } int Level::compare(const Level& l) const { int res; if ((res = ltype1 - l.ltype1)) return res; if ((res = l1 - l.l1)) return res; if ((res = ltype2 - l.ltype2)) return res; return l2 - l.l2; } static std::string describe_level(int ltype, int val) { char buf[256]; switch (ltype) { case 1: return "Ground or water surface"; case 2: return "Cloud base level"; case 3: return "Level of cloud tops"; case 4: return "Level of 0°C isotherm"; case 5: return "Level of adiabatic condensation lifted from the surface"; case 6: return "Maximum wind level"; case 7: return "Tropopause"; case 8: if (val == 0) return "Nominal top of atmosphere"; else return snprintf(buf, 256, "Nominal top of atmosphere, channel %d", val), buf; case 9: return "Sea bottom"; case 20: return snprintf(buf, 256, "Isothermal level, %.1fK", (double)val/10), buf; case 100: return snprintf(buf, 256, "Isobaric surface, %.2fhPa", (double)val/100), buf; case 101: return "Mean sea level"; case 102: return snprintf(buf, 256, "%.3fm above mean sea level", (double)val/1000), buf; case 103: return snprintf(buf, 256, "%.3fm above ground", (double)val/1000), buf; case 104: return snprintf(buf, 256, "Sigma level %.5f", (double)val/10000), buf; case 105: return snprintf(buf, 256, "Hybrid level %d", val), buf; case 106: return snprintf(buf, 256, "%.3fm below land surface", (double)val/1000), buf; case 107: return snprintf(buf, 256, "Isentropic (theta) level, potential temperature %.1fK", (double)val/10), buf; case 108: return snprintf(buf, 256, "Pressure difference %.2fhPa from ground to level", (double)val/100), buf; case 109: return snprintf(buf, 256, "Potential vorticity surface %.3f 10-6 K m2 kg-1 s-1", (double)val/1000), buf; case 111: return snprintf(buf, 256, "ETA* level %.5f", (double)val/10000), buf; case 117: return snprintf(buf, 256, "Mixed layer depth %.3fm", (double)val/1000), buf; case 160: return snprintf(buf, 256, "%.3fm below sea level", (double)val/1000), buf; case 200: return "Entire atmosphere (considered as a single layer)"; case 201: return "Entire ocean (considered as a single layer)"; case 204: return "Highest tropospheric freezing level"; case 206: return "Grid scale cloud bottom level"; case 207: return "Grid scale cloud top level"; case 209: return "Boundary layer cloud bottom level"; case 210: return "Boundary layer cloud top level"; case 211: return "Boundary layer cloud layer"; case 212: return "Low cloud bottom level"; case 213: return "Low cloud top level"; case 214: return "Low cloud layer"; case 215: return "Cloud ceiling"; case 220: return "Planetary Boundary Layer"; case 222: return "Middle cloud bottom level"; case 223: return "Middle cloud top level"; case 224: return "Middle cloud layer"; case 232: return "High cloud bottom level"; case 233: return "High cloud top level"; case 234: return "High cloud layer"; case 235: return snprintf(buf, 256, "Ocean Isotherm Level, %.1fK", (double)val/10), buf; case 240: return "Ocean Mixed Layer"; case 242: return "Convective cloud bottom level"; case 243: return "Convective cloud top level"; case 244: return "Convective cloud layer"; case 245: return "Lowest level of the wet bulb zero"; case 246: return "Maximum equivalent potential temperature level"; case 247: return "Equilibrium level"; case 248: return "Shallow convective cloud bottom level"; case 249: return "Shallow convective cloud top level"; case 251: return "Deep convective cloud bottom level"; case 252: return "Deep convective cloud top level"; case 253: return "Lowest bottom level of supercooled liquid water layer"; case 254: return "Highest top level of supercooled liquid water layer"; case 255: return "Missing"; case 256: return "Clouds"; case 258: switch (val) { case 0: return "General cloud group"; case 1: return "CL"; case 2: return "CM"; case 3: return "CH"; default: return snprintf(buf, 256, "%d %d", ltype, val), buf; } break; case 259: return snprintf(buf, 256, "Cloud group %d", val), buf; case 260: return snprintf(buf, 256, "Cloud drift group %d", val), buf; case 261: return snprintf(buf, 256, "Cloud elevation group %d", val), buf; case 262: return "Direction and elevation of clouds"; case 265: return snprintf(buf, 256, "Non-physical data level #%d", val), buf; case MISSING_INT: return "Information about the station that generated the data"; default: return snprintf(buf, 256, "%d %d", ltype, val), buf; break; } } std::string Level::describe() const { if (ltype2 == MISSING_INT) return describe_level(ltype1, l1); if (ltype1 == 256 || ltype1 == 264) { string lev1 = describe_level(ltype1, l1); string lev2 = describe_level(ltype2, l2); return lev1 + ", " + lev2; } else { string lev1 = describe_level(ltype1, l1); string lev2 = describe_level(ltype2, l2); return "Layer from [" + lev1 + "] to [" + lev2 + "]"; } } void Level::to_stream(std::ostream& out, const char* undef) const { if (ltype1 == MISSING_INT) out << undef; else out << ltype1; out << ","; if (l1 == MISSING_INT) out << undef; else out << l1; out << ","; if (ltype2 == MISSING_INT) out << undef; else out << ltype2; out << ","; if (l2 == MISSING_INT) out << undef; else out << l2; } std::string Level::to_string(const char* undef) const { string res; if (ltype1 == MISSING_INT) res += undef; else res += std::to_string(ltype1); res += ","; if (l1 == MISSING_INT) res += undef; else res += std::to_string(l1); res += ","; if (ltype2 == MISSING_INT) res += undef; else res += std::to_string(ltype2); res += ","; if (l2 == MISSING_INT) res += undef; else res += std::to_string(l2); return res; } void Level::to_csv(CSVWriter& out) const { if (ltype1 == MISSING_INT) out.add_value_empty(); else out.add_value(ltype1); if (l1 == MISSING_INT) out.add_value_empty(); else out.add_value(l1); if (ltype2 == MISSING_INT) out.add_value_empty(); else out.add_value(ltype2); if (l2 == MISSING_INT) out.add_value_empty(); else out.add_value(l2); } Level Level::cloud(int ltype2, int l2) { return Level(256, MISSING_INT, ltype2, l2); } int Level::print(FILE* out, const char* undef, const char* end) const { int res = 0; if (ltype1 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d,", ltype1); if (l1 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d,", l1); if (ltype2 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d,", ltype2); if (l2 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d", l2); res += fprintf(out, "%s", end); return res; } std::ostream& operator<<(std::ostream& out, const Level& l) { l.to_stream(out); return out; } /* * Trange */ bool Trange::is_missing() const { return pind == MISSING_INT && p1 == MISSING_INT && p2 == MISSING_INT; } int Trange::compare(const Trange& t) const { int res; if ((res = pind - t.pind)) return res; if ((res = p1 - t.p1)) return res; return p2 - t.p2; } bool Trange::operator==(const Trange& o) const { return std::tie(pind, p1, p2) == std::tie(o.pind, o.p1, o.p2); } bool Trange::operator!=(const Trange& o) const { return std::tie(pind, p1, p2) != std::tie(o.pind, o.p1, o.p2); } bool Trange::operator<(const Trange& o) const { return std::tie(pind, p1, p2) < std::tie(o.pind, o.p1, o.p2); } bool Trange::operator>(const Trange& o) const { return std::tie(pind, p1, p2) > std::tie(o.pind, o.p1, o.p2); } bool Trange::operator<=(const Trange& o) const { return std::tie(pind, p1, p2) <= std::tie(o.pind, o.p1, o.p2); } bool Trange::operator>=(const Trange& o) const { return std::tie(pind, p1, p2) >= std::tie(o.pind, o.p1, o.p2); } void Trange::to_stream(std::ostream& out, const char* undef) const { if (pind == MISSING_INT) out << undef; else out << pind; out << ","; if (p1 == MISSING_INT) out << undef; else out << p1; out << ","; if (p2 == MISSING_INT) out << undef; else out << p2; } std::string Trange::to_string(const char* undef) const { string res; if (pind == MISSING_INT) res += undef; else res += std::to_string(pind); res += ","; if (p1 == MISSING_INT) res += undef; else res += std::to_string(p1); res += ","; if (p2 == MISSING_INT) res += undef; else res += std::to_string(p2); return res; } void Trange::to_csv(CSVWriter& out) const { if (pind == MISSING_INT) out.add_value_empty(); else out.add_value(pind); if (p1 == MISSING_INT) out.add_value_empty(); else out.add_value(p1); if (p2 == MISSING_INT) out.add_value_empty(); else out.add_value(p2); } Trange Trange::instant() { return Trange(254, 0, 0); } static std::string format_seconds(int val) { static const int bufsize = 128; char buf[bufsize]; if (val == MISSING_INT) return "-"; int i = 0; if (val / (3600*24) != 0) { i += snprintf(buf+i, bufsize-i, "%dd ", val / (3600*24)); val = abs(val) % (3600*24); } if (val / 3600 != 0) { i += snprintf(buf+i, bufsize-i, "%dh ", val / 3600); val = abs(val) % 3600; } if (val / 60 != 0) { i += snprintf(buf+i, bufsize-i, "%dm ", val / 60); val = abs(val) % 60; } if (val) i += snprintf(buf+i, bufsize-i, "%ds ", val); if (i > 0) --i; else { buf[0] = '0'; i = 1; } buf[i] = 0; return buf; } static std::string mkdesc(const std::string& root, int p1, int p2) { if (p1 == MISSING_INT && p2 == MISSING_INT) return root; string res = root; if (p2 != MISSING_INT) res += " over " + format_seconds(p2); if (p1 == MISSING_INT) return res; if (p1 < 0) return res + format_seconds(-p1) + " before reference time"; return res + " at forecast time " + format_seconds(p1); } std::string Trange::describe() const { char buf[256]; switch (pind) { case 0: return mkdesc("Average", p1, p2); case 1: return mkdesc("Accumulation", p1, p2); case 2: return mkdesc("Maximum", p1, p2); case 3: return mkdesc("Minimum", p1, p2); case 4: return mkdesc("Difference (end minus beginning)", p1, p2); case 5: return mkdesc("Root Mean Square", p1, p2); case 6: return mkdesc("Standard Deviation", p1, p2); case 7: return mkdesc("Covariance (temporal variance)", p1, p2); case 8: return mkdesc("Difference (beginning minus end)", p1, p2); case 9: return mkdesc("Ratio", p1, p2); case 51: return mkdesc("Climatological Mean Value", p1, p2); case 200: return mkdesc("Vectorial mean", p1, p2); case 201: return mkdesc("Mode", p1, p2); case 202: return mkdesc("Standard deviation vectorial mean", p1, p2); case 203: return mkdesc("Vectorial maximum", p1, p2); case 204: return mkdesc("Vectorial minimum", p1, p2); case 205: return mkdesc("Product with a valid time ranging", p1, p2); case 254: if (p1 == 0 && p2 == 0) return "Analysis or observation, istantaneous value"; else return "Forecast at t+" + format_seconds(p1) + ", instantaneous value"; default: return snprintf(buf, 256, "%d %d %d", pind, p1, p2), buf; } } int Trange::print(FILE* out, const char* undef, const char* end) const { int res = 0; if (pind == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d,", pind); if (p1 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d,", p1); if (p2 == MISSING_INT) res += fprintf(out, "%s,", undef); else res += fprintf(out, "%d", p2); res += fprintf(out, "%s", end); return res; } std::ostream& operator<<(std::ostream& out, const Trange& l) { l.to_stream(out); return out; } /* * Ident */ Ident::Ident(const char* value) : value(value ? strdup(value) : nullptr) {} Ident::Ident(const std::string& value) : value(strndup(value.data(), value.size())) {} Ident::Ident(const Ident& o) : value(o.value ? strdup(o.value) : nullptr) {} Ident::Ident(Ident&& o) : value(o.value) { o.value = nullptr; } Ident::~Ident() { free(value); } Ident& Ident::operator=(const Ident& o) { if (value == o.value) return *this; free(value); value = o.value ? strdup(o.value) : nullptr; return *this; } Ident& Ident::operator=(Ident&& o) { if (value == o.value) return *this; free(value); if (o.value) { value = strdup(o.value); o.value = nullptr; } else value = nullptr; return *this; } Ident& Ident::operator=(const char* o) { if (value) free(value); value = o ? strdup(o) : nullptr; return *this; } Ident& Ident::operator=(const std::string& o) { if (value) free(value); value = strndup(o.c_str(), o.size()); return *this; } void Ident::clear() { free(value); value = 0; } int Ident::compare(const Ident& o) const { if (!value && !o.value) return 0; if (!value && o.value) return -1; if (value && !o.value) return 1; return strcmp(value, o.value); } int Ident::compare(const char* o) const { if (!value && !o) return 0; if (!value && o) return -1; if (value && !o) return 1; return strcmp(value, o); } int Ident::compare(const std::string& o) const { if (!value) return -1; return strcmp(value, o.c_str()); } bool Ident::is_missing() const { return value == nullptr; } Ident::operator std::string() const { if (!value) throw error_consistency("ident is not set"); return std::string(value); } std::ostream& operator<<(std::ostream& out, const Ident& i) { if (i.is_missing()) return out; else return out << (const char*)i; } /* * Station */ bool Station::is_missing() const { return report.empty() && coords.is_missing() && ident.is_missing(); } int Station::print(FILE* out, const char* end) const { int res = 0; if (coords.is_missing()) { fputs("(-,-) ", out); res += 6; } else res += coords.print(out, " "); if (ident.is_missing()) { putc('-', out); res += 1; } else { fputs(ident.get(), out); res += strlen(ident.get()); } res += fprintf(out, " %s%s", report.c_str(), end); return res; } std::string Station::to_string(const char* undef) const { string res = report; res += ","; res += coords.to_string(undef); res += ","; if (ident.is_missing()) res += undef; else res += ident.get(); return res; } std::ostream& operator<<(std::ostream& out, const Station& st) { return out << st.coords << "," << st.ident << "," << st.report; } /* * DBStation */ bool DBStation::is_missing() const { return Station::is_missing() && id == MISSING_INT; } int DBStation::print(FILE* out, const char* end) const { int res = 0; if (id == MISSING_INT) { fputs("- ", out); res += 2; } else res += fprintf(out, "%d,", id); res += Station::print(out, end); return res; } std::string DBStation::to_string(const char* undef) const { string res = report; res += ","; if (id == MISSING_INT) res += undef; else res += std::to_string(id); res += ","; res += coords.to_string(undef); res += ","; if (ident.is_missing()) res += undef; else res += ident.get(); return res; } std::ostream& operator<<(std::ostream& out, const DBStation& st) { if (st.id == MISSING_INT) out << "-,"; else out << st.id << ","; return out << (const Station&)st; } } namespace std { size_t hash::operator()(dballe::Level const& o) const noexcept { using dballe::MISSING_INT; size_t res = 0; if (o.ltype1 != MISSING_INT) res += o.ltype1; if (o.l1 != MISSING_INT) res += o.l1; if (o.ltype2 != MISSING_INT) res += o.ltype2 << 8; if (o.l2 != MISSING_INT) res += o.l2; return res; } size_t hash::operator()(dballe::Trange const& o) const noexcept { using dballe::MISSING_INT; size_t res = 0; if (o.pind != MISSING_INT) res += o.pind; if (o.p1 != MISSING_INT) res += o.p1; if (o.p2 != MISSING_INT) res += o.p2; return res; } size_t hash::operator()(dballe::Coords const& o) const noexcept { return o.lat xor o.lon; } size_t hash::operator()(dballe::Ident const& o) const noexcept { if (o.is_missing()) return 0; return std::hash{}(o.get()); } size_t hash::operator()(dballe::Station const& o) const noexcept { size_t res = std::hash{}(o.report); res += std::hash{}(o.coords); res += std::hash{}(o.ident); return res; } size_t hash::operator()(dballe::DBStation const& o) const noexcept { size_t res = std::hash{}(o.report); res += o.id; res += std::hash{}(o.coords); res += std::hash{}(o.ident); return res; } } dballe-8.6/dballe/value-test.cc0000644000175000017500000000051013554564112013360 00000000000000#include "core/tests.h" #include "value.h" using namespace std; using namespace wreport::tests; using namespace dballe; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("dballe_value"); void Tests::register_tests() { add_method("empty", []() { }); } } dballe-8.6/dballe/python.h0000644000175000017500000000562013554573614012470 00000000000000#ifndef DBALLE_PYTHON_H #define DBALLE_PYTHON_H #include #include #ifndef PyObject_HEAD // Forward-declare PyObjetc and PyTypeObject // see https://mail.python.org/pipermail/python-dev/2003-August/037601.html extern "C" { struct _object; typedef _object PyObject; struct _typeobject; typedef _typeobject PyTypeObject; } #endif extern "C" { /** * C++ functions exported by the wreport python bindings, to be used by other * C++ bindings. * * To use them, retrieve a pointer to the struct via the Capsule system: * \code * dbapy_c_api* wrpy = (dbapy_c_api*)PyCapsule_Import("_dballe._C_API", 0); * \endcode * */ struct dbapy_c_api { // API version 1.x /// C API major version (updated on incompatible changes) unsigned version_major; /// C API minor version (updated on backwards-compatible changes) unsigned version_minor; /// dballe.Message type PyTypeObject* message_type; /// Create a dballe.Message with a new empty message of the given type PyObject* (*message_create_new)(dballe::MessageType); /// Create a dballe.Message referencing the given message PyObject* (*message_create)(std::shared_ptr); #if 0 /// Create a new unset wreport.Var object PyObject* (*var_create)(const wreport::Varinfo&); /// Create a new wreport.Var object with an integer value PyObject* (*var_create_i)(const wreport::Varinfo&, int); /// Create a new wreport.Var object with a double value PyObject* (*var_create_d)(const wreport::Varinfo&, double); /// Create a new wreport.Var object with a C string value PyObject* (*var_create_c)(const wreport::Varinfo&, const char*); /// Create a new wreport.Var object with a std::string value PyObject* (*var_create_s)(const wreport::Varinfo&, const std::string&); /// Create a new wreport.Var object as a copy of an existing var PyObject* (*var_create_copy)(const wreport::Var&); /// Read the value of a variable as a new Python object PyObject* (*var_value_to_python)(const wreport::Var&); /// Set the value of a variable from a Python object (borrowed reference) int (*var_value_from_python)(PyObject* o, wreport::Var&); /// Create a wreport.Varinfo object from a C++ Varinfo PyObject* (*varinfo_create)(wreport::Varinfo); /// Create a wreport:Vartable object from a C++ Vartable PyObject* (*vartable_create)(const wreport::Vartable*); /// Vartable type PyTypeObject* vartable_type; /// Var type PyTypeObject* var_type; // API version 1.1 /// Create a new wreport.Var object, moving an existing var PyObject* (*var_create_move)(wreport::Var&&); /// Return the variable for a wreport.Var object wreport:: Var* (*var)(PyObject* o); /// Create a new wreport.Var object with the value from another variable PyObject* (*var_create_v)(const wreport::Varinfo&, const wreport::Var&); #endif }; } #endif dballe-8.6/dballe/cursor.cc0000644000175000017500000000005313554564112012606 00000000000000#include "cursor.h" namespace dballe { } dballe-8.6/dballe/db.cc0000644000175000017500000001056413554564112011666 00000000000000#include "db.h" #include "db/db.h" #include "sql/sql.h" #include "core/string.h" #include "wreport/utils/string.h" #include #include using namespace wreport; namespace dballe { static bool parse_wipe(const std::string& strval) { std::string val = str::lower(strval); if (val.empty()) return true; if (val == "1") return true; if (val == "yes") return true; if (val == "true") return true; if (val == "0") return false; if (val == "no") return false; if (val == "false") return false; wreport::error_consistency::throwf("unsupported value for wipe: %s (supported: 1/0, true/false, yes/no)", strval.c_str()); } void DBConnectOptions::reset_actions() { wipe = false; } std::unique_ptr DBConnectOptions::create(const std::string& url) { std::unique_ptr res(new DBConnectOptions); res->url = url; std::string wipe; if (url_pop_query_string(res->url, "wipe", wipe)) res->wipe = parse_wipe(wipe); else res->wipe = false; if (strncmp(url.c_str(), "test:", 5) == 0) { const char* envurl = getenv("DBA_DB"); if (!envurl) res->url = "sqlite://test.sqlite"; else res->url = envurl; } return res; } std::unique_ptr DBConnectOptions::test_create(const char* backend) { std::string envname = "DBA_DB"; if (backend) { envname += "_"; envname += backend; } const char* envurl = getenv(envname.c_str()); if (!envurl) envurl = "test:"; return create(envurl); } const DBImportOptions DBImportOptions::defaults; std::unique_ptr DBImportOptions::create() { return std::unique_ptr(new DBImportOptions); } const DBInsertOptions DBInsertOptions::defaults; std::unique_ptr DBInsertOptions::create() { return std::unique_ptr(new DBInsertOptions); } /* * Cursor* */ Cursor::~Cursor() { } /* * Transaction */ Transaction::~Transaction() {} void Transaction::import_messages(const std::vector>& messages, const DBImportOptions& opts) { for (const auto& i: messages) import_message(*i, opts); } /* * DB */ DB::~DB() { } std::shared_ptr DB::connect(const DBConnectOptions& opts) { if (opts.url == "mem:") { return db::DB::connect_memory(); } else { std::unique_ptr conn(sql::Connection::create(opts)); auto res = db::DB::create(move(conn)); if (opts.wipe) res->reset(); return res; } } std::unique_ptr DB::query_stations(const Query& query) { auto t = transaction(); auto res = t->query_stations(query); return res; } std::unique_ptr DB::query_station_data(const Query& query) { auto t = transaction(); auto res = t->query_station_data(query); return res; } std::unique_ptr DB::query_data(const Query& query) { auto t = transaction(); auto res = t->query_data(query); return res; } std::unique_ptr DB::query_summary(const Query& query) { auto t = transaction(); auto res = t->query_summary(query); return res; } std::unique_ptr DB::query_messages(const Query& query) { auto t = transaction(); auto res = t->query_messages(query); return res; } void DB::remove_all() { auto t = transaction(); t->remove_all(); t->commit(); } void DB::remove_station_data(const Query& query) { auto t = transaction(); t->remove_station_data(query); t->commit(); } void DB::remove_data(const Query& query) { auto t = transaction(); t->remove_data(query); t->commit(); } void DB::import_message(const Message& message, const DBImportOptions& opts) { auto t = transaction(); t->import_message(message, opts); t->commit(); } void DB::import_messages(const std::vector>& messages, const DBImportOptions& opts) { auto t = transaction(); t->import_messages(messages, opts); t->commit(); } void DB::insert_station_data(Data& vals, const DBInsertOptions& opts) { auto t = transaction(); t->insert_station_data(vals, opts); t->commit(); } void DB::insert_data(Data& vals, const DBInsertOptions& opts) { auto t = transaction(); t->insert_data(vals, opts); t->commit(); } } dballe-8.6/dballe/tests-main.cc0000644000175000017500000000246113554564112013362 00000000000000#include #include #include #include #include #include #include #include #include using namespace wreport::tests; void signal_to_exception(int) { throw std::runtime_error("killing signal catched"); } int main(int argc,const char* argv[]) { dballe::db::v7::Trace::set_in_test_suite(); signal(SIGSEGV, signal_to_exception); signal(SIGILL, signal_to_exception); auto& tests = TestRegistry::get(); wreport::term::Terminal output(stderr); std::unique_ptr controller; bool verbose = (bool)getenv("TEST_VERBOSE"); if (verbose) controller.reset(new VerboseTestController(output)); else controller.reset(new SimpleTestController(output)); if (const char* whitelist = getenv("TEST_WHITELIST")) controller->whitelist = whitelist; if (const char* blacklist = getenv("TEST_BLACKLIST")) controller->blacklist = blacklist; auto all_results = tests.run_tests(*controller); TestResultStats rstats(all_results); rstats.print_results(output); if (verbose) rstats.print_stats(output); rstats.print_summary(output); return rstats.success ? 0 : 1; } dballe-8.6/dballe/value.h0000644000175000017500000000505313554564112012254 00000000000000#ifndef DBALLE_VALUE_H #define DBALLE_VALUE_H #include #include #include #include namespace wreport { struct Var; } namespace dballe { /** * Container for a wreport::Var pointer */ class Value { protected: wreport::Var* m_var = nullptr; public: Value() = default; Value(const Value& o); Value(Value&& o) : m_var(o.m_var) { o.m_var = nullptr; } /// Construct from a wreport::Var Value(const wreport::Var& var); /// Construct from a wreport::Var, taking ownership of it Value(std::unique_ptr&& var) : m_var(var.release()) {} ~Value(); Value& operator=(const Value& o); Value& operator=(Value&& o); bool operator==(const Value& o) const; bool operator!=(const Value& o) const; const wreport::Var* get() const { return m_var; } wreport::Var* get() { return m_var; } const wreport::Var* operator->() const { return m_var; } wreport::Var* operator->() { return m_var; } const wreport::Var& operator*() const { return *m_var; } wreport::Var& operator*() { return *m_var; } /// Return the varcode of the variable, or 0 if no variable has been set wreport::Varcode code() const; /// Fill from a wreport::Var void reset(const wreport::Var& var); /// Fill from a wreport::Var, taking ownership of it void reset(std::unique_ptr&& var); /// Return the Var pointer, setting the Value to undefined std::unique_ptr release(); /// Print the contents of this Value void print(FILE* out) const; }; /** * Container for a wreport::Var pointer, and its database ID */ struct DBValue : public Value { using Value::Value; /// Database ID of the value int data_id = MISSING_INT; DBValue() = default; DBValue(const DBValue& o) = default; DBValue(DBValue&& o) = default; /// Construct from a wreport::Var DBValue(int data_id, const wreport::Var& var) : Value(var), data_id(data_id) {} /// Construct from a wreport::Var, taking ownership of it DBValue(int data_id, std::unique_ptr&& var) : Value(std::move(var)), data_id(data_id) {} DBValue& operator=(const DBValue&) = default; DBValue& operator=(DBValue&&) = default; bool operator==(const DBValue& o) const; bool operator!=(const DBValue& o) const; /// Print the contents of this Value void print(FILE* out) const; }; std::ostream& operator<<(std::ostream&, const Value&); std::ostream& operator<<(std::ostream&, const DBValue&); } #endif dballe-8.6/dballe/types.h0000644000175000017500000006431413572457034012315 00000000000000#ifndef DBALLE_TYPES_H #define DBALLE_TYPES_H /** @file * Common base types used by most of DB-All.e code. */ #include #include #include #include #include namespace wreport { class Var; } namespace dballe { struct CSVWriter; /** * Calendar date. * * If year is 0xffff, then all the date is considered missing. Else, all fields * must be set. */ struct Date { unsigned short year; unsigned char month; unsigned char day; /// Construct a missing date Date(); /** * Construct from broken down values. * * A year of MISSING_INT constructs a missing Date. In any other case, * arguments are validated with Date::validate(). */ Date(int ye, int mo=1, int da=1); /// Copy constructor Date(const Date& d) = default; /// Create a date from a Julian day static Date from_julian(int jday); /// Check if this date is the missing value bool is_missing() const; /// Convert the date to Julian day int to_julian() const; /** * Write the date to an output stream in ISO8601 date format. */ void to_stream_iso8601(std::ostream& out) const; /** * Write the date as a CSV field in ISO8601 date format. */ void to_csv_iso8601(CSVWriter& out) const; /** * Generic comparison * * Returns a negative number if *this < other * Returns zero if *this == other * Returns a positive number if *this > other */ int compare(const Date& other) const; bool operator<(const Date& dt) const; bool operator>(const Date& dt) const; bool operator==(const Date& dt) const; bool operator!=(const Date& dt) const; /// Raise an exception if the three values do not represent a valid date static void validate(int ye, int mo, int da); /// Return the number of days in the given month static int days_in_month(int year, int month); /// Convert a calendar date into a Julian day static int calendar_to_julian(int year, int month, int day); /// Convert a Julian day into a calendar date static void julian_to_calendar(int jday, unsigned short& year, unsigned char& month, unsigned char& day); }; std::ostream& operator<<(std::ostream& out, const Date& dt); /** * Time of the day. * * If hour is 0xff, then all the time is considered missing. Else, all fields * must be set. */ struct Time { unsigned char hour; unsigned char minute; unsigned char second; /// Construct a missing time Time(); /** * Construct from broken down values. * * A hour of MISSING_INT constructs a missing Time. In any other case, * arguments are validated with Time::validate(). */ Time(int ho, int mi=0, int se=0); Time(const Time& t) = default; /// Check if this time is the missing value bool is_missing() const; /** * Write the time to an output stream in ISO8601 extended format * (hh:mm:ss). */ void to_stream_iso8601(std::ostream& out) const; /** * Write the time as a CSV field in ISO8601 date format. */ void to_csv_iso8601(CSVWriter& out) const; /** * Generic comparison * * Returns a negative number if *this < other * Returns zero if *this == other * Returns a positive number if *this > other */ int compare(const Time& other) const; bool operator<(const Time& dt) const; bool operator>(const Time& dt) const; bool operator==(const Time& dt) const; bool operator!=(const Time& dt) const; /** * Raise an exception if the three values do not represent a valid time. * * A value of 23:59:60 is allowed to accomodate for times during leap * seconds. */ static void validate(int ho, int mi, int se); }; std::ostream& operator<<(std::ostream& out, const Time& t); /** * Date and time * * If year is 0xffff, then all the datetime is considered missing. Else, all * fields must be set. */ struct Datetime { unsigned short year; unsigned char month; unsigned char day; unsigned char hour; unsigned char minute; unsigned char second; /// Construct a missing datetime Datetime(); Datetime(const Date& date, const Time& time); /** * Construct from broken down values. * * A year of MISSING_INT constructs a missing Datetime. In any other case, * arguments are validated with Datetime::validate(). */ Datetime(int ye, int mo=1, int da=1, int ho=0, int mi=0, int se=0); /// Set the date from a Julian day static Datetime from_julian(int jday, int ho=0, int mi=0, int se=0); /** * Return a Datetime filling the parts set to MISSING_INT with their lowest * possible values */ static Datetime lower_bound(int ye, int mo, int da, int ho, int mi, int se); /** * Return a Datetime filling the parts set to MISSING_INT with their * highest possible values */ static Datetime upper_bound(int ye, int mo, int da, int ho, int mi, int se); /// Fill possibly missing fields with their lowest valid value void set_lower_bound(); /// Fill possibly missing fields with their highest valid value void set_upper_bound(); /// Return a Date with this date Date date() const; /// Return a Time with this time Time time() const; /// Check if this datetime is the missing value bool is_missing() const; /// Convert the date to Julian day int to_julian() const; /** * Generic comparison * * Returns a negative number if *this < other * Returns zero if *this == other * Returns a positive number if *this > other */ int compare(const Datetime& other) const; bool operator==(const Datetime& o) const; bool operator!=(const Datetime& o) const; bool operator<(const Datetime& o) const; bool operator>(const Datetime& o) const; bool operator<=(const Datetime& o) const; bool operator>=(const Datetime& o) const; /** * Print to an output stream in ISO8601 combined format. */ int print_iso8601(FILE* out, char sep='T', const char* end="\n") const; /** * Print to an output stream in ISO8601 combined format. */ int print(FILE* out, const char* end="\n") const; /** * Write the datetime to an output stream in ISO8601 combined format. * * @param out the output stream * @param sep the separator character between date and time * @param tz the timezone string to use at the end of the datetime */ void to_stream_iso8601(std::ostream& out, char sep='T', const char* tz="") const; /** * Write the datetime as a CSV field in ISO8601 date format. */ void to_csv_iso8601(CSVWriter& out, char sep='T', const char* tz="") const; /// Write to a string in ISO8601 combined format std::string to_string(char sep='T', const char* tz="") const; /** * Parse an ISO8601 datetime string. * * Both 'T' and ' ' are allowed as separators. */ static Datetime from_iso8601(const char* str); /** * Raise an exception if the three values do not represent a valid * date/time. * * A value of 23:59:60 is allowed to accomodate for times during leap * seconds, but no effort is made to check if there has been a leap second * on the given date. */ static void validate(int ye, int mo, int da, int ho, int mi, int se); /** * Convert a datetime with an hour of 24:00:00 to hour 00:00:00 of the * following day. */ static void normalise_h24(int& ye, int& mo, int& da, int& ho, int& mi, int& se); }; std::ostream& operator<<(std::ostream& out, const Datetime& dt); /** * Range of datetimes. * * The range includes the extremes. A missing extreme in the range means an * open ended range. */ struct DatetimeRange { /// Lower bound of the range Datetime min; /// Upper bound of the range Datetime max; DatetimeRange() = default; DatetimeRange(const Datetime& min, const Datetime& max) : min(min), max(max) {} DatetimeRange( int yemin, int momin, int damin, int homin, int mimin, int semin, int yemax, int momax, int damax, int homax, int mimax, int semax); /// Check if this range is open on both sides bool is_missing() const; bool operator==(const DatetimeRange& o) const; bool operator!=(const DatetimeRange& o) const; bool operator<(const DatetimeRange& o) const; bool operator<=(const DatetimeRange& o) const; bool operator>(const DatetimeRange& o) const; bool operator>=(const DatetimeRange& o) const; /// Set the extremes void set(const Datetime& min, const Datetime& max); /** * Set the extremes from broken down components. * * If yemin or yemax are MISSING_INT, they are taken as an open ended range * boundary. * * If any other *min values are MISSING_INT, they are filled with the * lowest possible valid value they can have. * * If any other *max values are MISSING_INT, they are filled with the * highest possible valid value they can have. */ void set(int yemin, int momin, int damin, int homin, int mimin, int semin, int yemax, int momax, int damax, int homax, int mimax, int semax); /** * Merge \a range into this one, resulting in the smallest range that * contains both. */ void merge(const DatetimeRange& range); /// Check if a Datetime is inside this range bool contains(const Datetime& dt) const; /// Check if a range is inside this range (extremes included) bool contains(const DatetimeRange& dtr) const; /// Check if the two ranges are completely disjoint bool is_disjoint(const DatetimeRange& dtr) const; /// Print to an output stream in ISO8601 combined format. int print(FILE* out, const char* end="\n") const; }; std::ostream& operator<<(std::ostream& out, const DatetimeRange& dtr); /** * Coordinates * * When given as an integer, a latitude/longitude value is intended in 1/100000 * of a degree, which is the maximum resolution supported by DB-All.e. * * When given as a double a latitude/longitude value is intended to be in * degrees. * * Longitude values are normalized between -180.0 and 180.0. */ struct Coords { /// Latitude in 1/100000 of a degree (5 significant digits preserved) int lat = MISSING_INT; /** * Longitude in 1/100000 of a degree (5 significant digits preserved) and * normalised between -180.0 and 180.0. */ int lon = MISSING_INT; /// Construct a missing Coords Coords() = default; /// Construct a coords from integers in 1/100000 of a degree Coords(int lat, int lon); /// Construct a coords from values in degrees Coords(double lat, double lon); /// Check if these coordinates are undefined bool is_missing() const; /// Set the latitude only void set_lat(double lat); /// Set the longitude only void set_lon(double lon); /// Set the latitude only, in 1/100000 of a degree void set_lat(int lat); /// Set the longitude only, in 1/100000 of a degree void set_lon(int lon); /// Set from integers in 1/100000 of a degree void set(int lat, int lon); /// Set from values in degrees void set(double lat, double lon); /// Get the latitude in degrees double dlat() const; /// Get the longitude in degrees double dlon() const; /** * Compare two Coords strutures, for use in sorting. * * Sorting happens by latitude first, then by longitude. It is implemented * mainly for the purpose of being able to use Coords as a key in a sorted * container. * * @return * -1 if *this < o, 0 if *this == o, 1 if *this > o */ int compare(const Coords& o) const; bool operator==(const Coords& o) const; bool operator!=(const Coords& o) const; bool operator<(const Coords& o) const; bool operator>(const Coords& o) const; bool operator<=(const Coords& o) const; bool operator>=(const Coords& o) const; /// Print to an output stream int print(FILE* out, const char* end="\n") const; /// Format to a string std::string to_string(const char* undef="-") const; /// Convert a latitude to the internal integer representation static int lat_to_int(double lat); /// Convert a longitude to the internal integer representation static int lon_to_int(double lat); /// Convert a latitude from the internal integer representation static double lat_from_int(int lat); /// Convert a longitude from the internal integer representation static double lon_from_int(int lon); }; std::ostream& operator<<(std::ostream&, const Coords&); /** * Range of latitudes. * * When given as an integer, a latitude value is intended in 1/100000 of a * degree, which is the maximum resolution supported by DB-All.e. * * When given as a double a latitude value is intended to be in degrees. * * Values are matched between imin and imax, both extremes are considered part * of the range. * * Invariant: imin <= imax. */ struct LatRange { /// Minimum possible integer value static constexpr int IMIN = -9000000; /// Maximum possible integer value static constexpr int IMAX = 9000000; /// Minimum possible double value static constexpr double DMIN = -90.0; /// Maximum possible double value static constexpr double DMAX = 90.0; /// Minimum latitude int imin = IMIN; /// Maximum latitude int imax = IMAX; /// Construct a LatRange matching any latitude LatRange() = default; /// Construct a LatRange given integer extremes LatRange(int min, int max); /// Construct a LatRange given extremes in degrees LatRange(double min, double max); bool operator==(const LatRange& lr) const; bool operator!=(const LatRange& lr) const; /// Return true if the LatRange matches any latitude bool is_missing() const; /// Get the lower extreme as double double dmin() const; /// Get the upper extreme as double double dmax() const; /// Get the extremes as double void get(double& min, double& max) const; /// Set the extremes as integers void set(int min, int max); /// Set the extremes in degrees void set(double min, double max); /// Check if a point is inside this range (extremes included) bool contains(int lat) const; /// Check if a point is inside this range (extremes included) bool contains(double lat) const; /// Check if a range is inside this range (extremes included) bool contains(const LatRange& lr) const; /** * Print the LatRange to a FILE*. * * @param out The output stream * @param end String to print after the Station */ int print(FILE* out, const char* end="\n") const; }; std::ostream& operator<<(std::ostream& out, const LatRange& lr); /** * Range of longitudes. * * When given as an integer, a longitude value is intended in 1/100000 of a * degree, which is the maximum resolution supported by DB-All.e. * * When given as a double a longitude value is intended to be in degrees. * * Longitude values are normalized between -180.0 and 180.0. The range is the * angle that goes from imin to imax. Both extremes are considered part of the * range. * * A range that matches any longitude has both imin and imax set to * MISSING_INT. * * Invariant: if imin == MISSING_INT, then imax == MISSING_INT. An open-ended * longitude range makes no sense, since longitudes move alongide a closed * circle. */ struct LonRange { /// Initial point of the longitude range int imin = MISSING_INT; /// Final point of the longitude range int imax = MISSING_INT; /// Construct a range that matches any longitude LonRange() = default; /// Construct a range given integer extremes LonRange(int min, int max); /// Construct a range given extremes in degrees LonRange(double min, double max); bool operator==(const LonRange& lr) const; bool operator!=(const LonRange& lr) const; /// Return true if the LonRange matches any longitude bool is_missing() const; /// Get the lower extreme as double, or -180.0 if missing double dmin() const; /// Get the upper extreme as double, or 180.0 if missing double dmax() const; /** * Get the extremes in degrees. * * If is_missing() == true, it returns the values -180.0 and 180.0 */ void get(double& min, double& max) const; /** * Set the extremes as integers, throwing an error in case of open ended * ranges */ void set(int min, int max); /** * Set the extremes in degrees, throwing an error in case of open ended * ranges */ void set(double min, double max); /** * Set from another LonRange, throwing an error in case of open ended * ranges */ void set(const LonRange& lr); /// Check if a point is inside this range (extremes included) bool contains(int lon) const; /// Check if a point is inside this range (extremes included) bool contains(double lon) const; /// Check if a range is inside this range (extremes included) bool contains(const LonRange& lr) const; /** * Print the LonRange to a FILE*. * * @param out The output stream * @param end String to print after the Station */ int print(FILE* out, const char* end="\n") const; }; std::ostream& operator<<(std::ostream& out, const LonRange& lr); /// Vertical level or layer struct Level { /// Type of the level or the first layer int ltype1; /// L1 value of the level or the first layer int l1; /// Type of the the second layer int ltype2; /// L2 value of the second layer int l2; Level(int ltype1=MISSING_INT, int l1=MISSING_INT, int ltype2=MISSING_INT, int l2=MISSING_INT) : ltype1(ltype1), l1(l1), ltype2(ltype2), l2(l2) {} /// Check if this level is fully set to the missing value bool is_missing() const; bool operator==(const Level& o) const; bool operator!=(const Level& o) const; bool operator<(const Level& o) const; bool operator>(const Level& o) const; bool operator<=(const Level& o) const; bool operator>=(const Level& o) const; /** * Generic comparison * * Returns a negative number if *this < other * Returns zero if *this == other * Returns a positive number if *this > other */ int compare(const Level& l) const; /** * Return a string description of this level */ std::string describe() const; /// Format to an output stream void to_stream(std::ostream& out, const char* undef="-") const; /// Format to a string std::string to_string(const char* undef="-") const; /** * Write the datetime to a CSV writer as 4 fields */ void to_csv(CSVWriter& out) const; /// Create a cloud special level static Level cloud(int ltype2=MISSING_INT, int l2=MISSING_INT); /// Print to an output stream int print(FILE* out, const char* undef="-", const char* end="\n") const; }; std::ostream& operator<<(std::ostream& out, const Level& l); /** * Information on how a value has been sampled or computed with regards to time. */ struct Trange { /// Time range type indicator. int pind; /// Time range P1 indicator. int p1; /// Time range P2 indicator. int p2; Trange(int pind=MISSING_INT, int p1=MISSING_INT, int p2=MISSING_INT) : pind(pind), p1(p1), p2(p2) {} /// Check if this level is fully set to the missing value bool is_missing() const; /** * Generic comparison * * Returns a negative number if *this < other * Returns zero if *this == other * Returns a positive number if *this > other */ int compare(const Trange& t) const; bool operator==(const Trange& o) const; bool operator!=(const Trange& o) const; bool operator<(const Trange& o) const; bool operator>(const Trange& o) const; bool operator<=(const Trange& o) const; bool operator>=(const Trange& o) const; /** * Return a string description of this time range */ std::string describe() const; /// Format to an output stream void to_stream(std::ostream& out, const char* undef="-") const; /// Format to a string std::string to_string(const char* undef="-") const; /** * Write the datetime to a CSV writer as 3 fields */ void to_csv(CSVWriter& out) const; /// Time range for instant values static Trange instant(); /// Print to an output stream int print(FILE* out, const char* undef="-", const char* end="\n") const; }; std::ostream& operator<<(std::ostream& out, const Trange& l); /** * A station identifier, that can be any string (including the empty string) or * a missing value. */ class Ident { protected: char* value = nullptr; public: Ident() = default; Ident(const char* value); Ident(const std::string& value); Ident(const Ident& o); Ident(Ident&& o); ~Ident(); Ident& operator=(const Ident& o); Ident& operator=(Ident&& o); Ident& operator=(const char* o); Ident& operator=(const std::string& o); /// Get the string value (might be nullptr in case of missing value) const char* get() const { return value; } /// Set to missing value void clear(); int compare(const Ident& o) const; int compare(const char* o) const; int compare(const std::string& o) const; template bool operator==(const T& o) const { return compare(o) == 0; } template bool operator!=(const T& o) const { return compare(o) != 0; } template bool operator<(const T& o) const { return compare(o) < 0; } template bool operator<=(const T& o) const { return compare(o) <= 0; } template bool operator>(const T& o) const { return compare(o) > 0; } template bool operator>=(const T& o) const { return compare(o) >= 0; } /// Check if the Ident is set to the missing value bool is_missing() const; operator const char*() const { return value; } operator std::string() const; }; std::ostream& operator<<(std::ostream&, const Ident&); /** * Station information */ struct Station { /// Report name for this station std::string report; /// Station coordinates Coords coords; /// Mobile station identifier Ident ident; Station() = default; /// Return true if all the station fields are empty bool is_missing() const; bool operator==(const Station& o) const { return std::tie(report, coords, ident) == std::tie(o.report, o.coords, o.ident); } bool operator!=(const Station& o) const { return std::tie(report, coords, ident) != std::tie(o.report, o.coords, o.ident); } bool operator<(const Station& o) const { return std::tie(report, coords, ident) < std::tie(o.report, o.coords, o.ident); } bool operator<=(const Station& o) const { return std::tie(report, coords, ident) <= std::tie(o.report, o.coords, o.ident); } bool operator>(const Station& o) const { return std::tie(report, coords, ident) > std::tie(o.report, o.coords, o.ident); } bool operator>=(const Station& o) const { return std::tie(report, coords, ident) >= std::tie(o.report, o.coords, o.ident); } /** * Print the Station to a FILE*. * * @param out The output stream * @param end String to print after the Station */ int print(FILE* out, const char* end="\n") const; /// Format to a string std::string to_string(const char* undef="-") const; }; std::ostream& operator<<(std::ostream&, const Station&); struct DBStation : public Station { /** * Database ID of the station. * * It will be filled when the Station is inserted on the database. */ int id = MISSING_INT; DBStation() = default; /// Return true if all the station fields are empty bool is_missing() const; bool operator==(const DBStation& o) const { return std::tie(id, report, coords, ident) == std::tie(o.id, o.report, o.coords, o.ident); } bool operator!=(const DBStation& o) const { return std::tie(id, report, coords, ident) != std::tie(o.id, o.report, o.coords, o.ident); } bool operator<(const DBStation& o) const { return std::tie(id, report, coords, ident) < std::tie(o.id, o.report, o.coords, o.ident); } bool operator<=(const DBStation& o) const { return std::tie(id, report, coords, ident) <= std::tie(o.id, o.report, o.coords, o.ident); } bool operator>(const DBStation& o) const { return std::tie(id, report, coords, ident) > std::tie(o.id, o.report, o.coords, o.ident); } bool operator>=(const DBStation& o) const { return std::tie(id, report, coords, ident) >= std::tie(o.id, o.report, o.coords, o.ident); } /** * Print the Station to a FILE*. * * @param out The output stream * @param end String to print after the Station */ int print(FILE* out, const char* end="\n") const; /// Format to a string std::string to_string(const char* undef="-") const; }; std::ostream& operator<<(std::ostream&, const DBStation&); } namespace std { template<> struct hash { typedef dballe::Level argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; template<> struct hash { typedef dballe::Trange argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; template<> struct hash { typedef dballe::Coords argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; template<> struct hash { typedef dballe::Ident argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; template<> struct hash { typedef dballe::Station argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; template<> struct hash { typedef dballe::DBStation argument_type; typedef size_t result_type; result_type operator()(argument_type const& o) const noexcept; }; } #endif dballe-8.6/dballe/db-test.cc0000644000175000017500000000657513554564112012652 00000000000000#include "dballe/core/tests.h" #include "db.h" #include using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("db"); void Tests::register_tests() { add_method("dbconnectoptions", []{ auto opts = DBConnectOptions::create("sqlite://test.sqlite"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_false(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); opts = DBConnectOptions::create("postgresql:///testuser@testhost/testdb?port=5433"); wassert(actual(opts->url) == "postgresql:///testuser@testhost/testdb?port=5433"); wassert_false(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); opts = DBConnectOptions::create("mysql://host/db?user=testuser&password=testpass"); wassert(actual(opts->url) == "mysql://host/db?user=testuser&password=testpass"); wassert_false(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=yes"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); opts = DBConnectOptions::create("postgresql:///testuser@testhost/testdb?port=5433&wipe=yes"); wassert(actual(opts->url) == "postgresql:///testuser@testhost/testdb?port=5433"); wassert_true(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); opts = DBConnectOptions::create("mysql://host/db?wipe=yes&user=testuser&password=testpass"); wassert(actual(opts->url) == "mysql://host/db?user=testuser&password=testpass"); wassert_true(opts->wipe); opts->reset_actions(); wassert_false(opts->wipe); }); add_method("parse_wipe", []{ auto opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=yes"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe="); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=true"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=tRUe"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=1"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_true(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=false"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_false(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=0"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_false(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=no"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_false(opts->wipe); opts = DBConnectOptions::create("sqlite://test.sqlite?wipe=NO"); wassert(actual(opts->url) == "sqlite://test.sqlite"); wassert_false(opts->wipe); }); } } dballe-8.6/dballe/cursor.h0000644000175000017500000000546713572414716012472 00000000000000#ifndef DBALLE_CURSOR_H #define DBALLE_CURSOR_H #include #include #include #include #include namespace dballe { /** * Base class for cursors that iterate over DB query results */ class Cursor { public: virtual ~Cursor(); /** * Check if the cursor points to a valid value. * * @returns true if the cursor points to a valid accessible value, false if * next() has not been called yet, or if at the end of iteration * (i.e. next() returned false) */ virtual bool has_value() const = 0; /** * Get the number of rows still to be fetched * * @return * The number of rows still to be queried. The value is undefined if no * query has been successfully peformed yet using this cursor. */ virtual int remaining() const = 0; /** * Get a new item from the results of a query * * @returns * true if a new record has been read, false if there is no more data to read */ virtual bool next() = 0; /// Discard the results that have not been read yet virtual void discard() = 0; /** * Get the whole station data in a single call */ virtual DBStation get_station() const = 0; }; /// Cursor iterating over stations class CursorStation : public Cursor { public: /** * Get the station data values */ virtual DBValues get_values() const = 0; }; /// Cursor iterating over station data values class CursorStationData : public Cursor { public: /// Get the variable code virtual wreport::Varcode get_varcode() const = 0; /// Get the variable virtual wreport::Var get_var() const = 0; }; /// Cursor iterating over data values class CursorData : public Cursor { public: /// Get the variable code virtual wreport::Varcode get_varcode() const = 0; /// Get the variable virtual wreport::Var get_var() const = 0; /// Get the level virtual Level get_level() const = 0; /// Get the time range virtual Trange get_trange() const = 0; /// Get the datetime virtual Datetime get_datetime() const = 0; }; /// Cursor iterating over summary entries class CursorSummary : public Cursor { public: /// Get the level virtual Level get_level() const = 0; /// Get the time range virtual Trange get_trange() const = 0; /// Get the variable code virtual wreport::Varcode get_varcode() const = 0; /// Get the datetime range virtual DatetimeRange get_datetimerange() const = 0; /// Get the count of elements virtual size_t get_count() const = 0; }; /// Cursor iterating over messages class CursorMessage : public Cursor { public: virtual const Message& get_message() const = 0; virtual std::unique_ptr detach_message() = 0; }; } #endif dballe-8.6/dballe/data-test.cc0000644000175000017500000000054013554564112013160 00000000000000#include "core/tests.h" #include "data.h" using namespace std; using namespace wreport::tests; using namespace dballe; using namespace wreport; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("dballe_data"); void Tests::register_tests() { add_method("empty", []() { }); } } dballe-8.6/dballe/message-test.cc0000644000175000017500000001262113554564112013676 00000000000000#include "msg/tests.h" #include "message.h" using namespace std; using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("dballe_message"); void Tests::register_tests() { add_method("create", []() { auto msg = Message::create(MessageType::AMDAR); msg->set("year", newvar("B04001", 2009)); msg->set("month", newvar("B04002", 2)); msg->set("day", newvar("B04003", 24)); msg->set("hour", newvar("B04004", 11)); msg->set("minute", newvar("B04005", 31)); msg->set("ident", newvar("B01011", "EU3375")); msg->set("latitude", newvar("B05001", 48.90500)); msg->set("longitude", newvar("B06001", 10.63667)); Level l(102, 6260000); Trange t = Trange::instant(); msg->set(l, t, var("B01006", "LH968")); msg->set(l, t, var("B02061", 0)); msg->set(l, t, var("B02062", 3)); msg->set(l, t, var("B02064", 0)); msg->set(l, t, var("B07030", 6260.0)); msg->set(l, t, var("B08004", 3)); msg->set(l, t, var("B11001", 33)); msg->set(l, t, var("B11002", 33.4)); msg->set(l, t, var("B12101", 240.0)); msg->set(l, t, var("B13002", 0.0)); auto msgs = read_msgs("bufr/gts-acars-uk1.bufr", Encoding::BUFR); wassert(actual(msg->diff(*msgs[0])) == 0); }); add_method("get", []() { auto msgs = read_msgs("bufr/gts-acars-uk1.bufr", Encoding::BUFR); wassert(actual(msgs[0]->get_type()) == MessageType::AMDAR); wassert(actual(msgs[0]->get_datetime()) == Datetime(2009, 2, 24, 11, 31)); wassert(actual(msgs[0]->get_coords()) == Coords(48.90500, 10.63667)); wassert(actual(msgs[0]->get_ident()) == "EU3375"); wassert(actual(msgs[0]->get_report()) == "amdar"); }); add_method("get_shortcut", []() { auto msgs = read_msgs("bufr/gts-acars-uk1.bufr", Encoding::BUFR); const wreport::Var* var = msgs[0]->get("ident"); wassert(actual(var).istrue()); wassert(actual(*var) == dballe::var("B01011", "EU3375")); }); add_method("foreach_var", []() { auto msgs = read_msgs("bufr/gts-acars-uk1.bufr", Encoding::BUFR); struct Result { Level l; Trange t; wreport::Var v; Result(const Level& l, const Trange& t, const wreport::Var& v) : l(l), t(t), v(v) {} }; std::vector results; msgs[0]->foreach_var([&](const Level& l, const Trange& t, const wreport::Var& v) { results.emplace_back(l, t, v); return true; }); wassert(actual(results.size()) == 18u); wassert(actual(results[0].l) == Level()); wassert(actual(results[0].t) == Trange()); wassert(actual(results[0].v) == var("B01011", "EU3375")); wassert(actual(results[1].l) == Level()); wassert(actual(results[1].t) == Trange()); wassert(actual(results[1].v) == var("B04001", 2009)); wassert(actual(results[2].l) == Level()); wassert(actual(results[2].t) == Trange()); wassert(actual(results[2].v) == var("B04002", 2)); wassert(actual(results[3].l) == Level()); wassert(actual(results[3].t) == Trange()); wassert(actual(results[3].v) == var("B04003", 24)); wassert(actual(results[4].l) == Level()); wassert(actual(results[4].t) == Trange()); wassert(actual(results[4].v) == var("B04004", 11)); wassert(actual(results[5].l) == Level()); wassert(actual(results[5].t) == Trange()); wassert(actual(results[5].v) == var("B04005", 31)); wassert(actual(results[6].l) == Level()); wassert(actual(results[6].t) == Trange()); wassert(actual(results[6].v) == var("B05001", 48.90500)); wassert(actual(results[7].l) == Level()); wassert(actual(results[7].t) == Trange()); wassert(actual(results[7].v) == var("B06001", 10.63667)); wassert(actual(results[8].l) == Level(102, 6260000)); wassert(actual(results[8].t) == Trange::instant()); wassert(actual(results[8].v) == var("B01006", "LH968")); wassert(actual(results[9].l) == Level(102, 6260000)); wassert(actual(results[9].t) == Trange::instant()); wassert(actual(results[9].v) == var("B02061", 0)); wassert(actual(results[10].l) == Level(102, 6260000)); wassert(actual(results[10].t) == Trange::instant()); wassert(actual(results[10].v) == var("B02062", 3)); wassert(actual(results[11].l) == Level(102, 6260000)); wassert(actual(results[11].t) == Trange::instant()); wassert(actual(results[11].v) == var("B02064", 0)); wassert(actual(results[12].l) == Level(102, 6260000)); wassert(actual(results[12].t) == Trange::instant()); wassert(actual(results[12].v) == var("B07030", 6260.0)); wassert(actual(results[13].l) == Level(102, 6260000)); wassert(actual(results[13].t) == Trange::instant()); wassert(actual(results[13].v) == var("B08004", 3)); wassert(actual(results[14].l) == Level(102, 6260000)); wassert(actual(results[14].t) == Trange::instant()); wassert(actual(results[14].v) == var("B11001", 33)); wassert(actual(results[15].l) == Level(102, 6260000)); wassert(actual(results[15].t) == Trange::instant()); wassert(actual(results[15].v) == var("B11002", 33.4)); wassert(actual(results[16].l) == Level(102, 6260000)); wassert(actual(results[16].t) == Trange::instant()); wassert(actual(results[16].v) == var("B12101", 240.0)); wassert(actual(results[17].l) == Level(102, 6260000)); wassert(actual(results[17].t) == Trange::instant()); wassert(actual(results[17].v) == var("B13002", 0.0)); }); } } dballe-8.6/dballe/mklookup0000755000175000017500000002006013554564112012551 00000000000000#!/usr/bin/env python3 # Expand a switch in this format: # switch (key) { // mklookup # case "string1": code1; # case "string2": code2; # default: code3; # } # Into code that matches the given strings. # break; statements in each case are ignored, and effectively added if missing. import argparse import logging import sys import re import os from collections import defaultdict from contextlib import contextmanager log = logging.getLogger("mklookup") class Fail(Exception): pass class Builder: def __init__(self, indent, indent_step=4, file=sys.stdout): self.indent = indent self.indent_step = indent_step self.file = file @contextmanager def nest(self, indent=None): if indent is None: indent = self.indent_step yield Builder(self.indent + indent, file=self.file) @contextmanager def print_switch(self, condition): self.print("switch ({}) {{".format(condition)) yield Builder(self.indent + self.indent_step, file=self.file) self.print("}") @contextmanager def print_case(self, condition, unquoted=False): if unquoted: self.print("case {}:".format(condition)) else: self.print("case '{}':".format(condition)) sub = Builder(self.indent + self.indent_step, file=self.file) yield sub sub.print("break;") def print_key_match(self, pos, val): if len(val) == 1: self.print("if (key[{}] == '{}') {{".format(pos, val)) else: self.print('if (memcmp(key + {}, "{}", {}) == 0) {{'.format(pos, val, len(val))) def print_leaf_check(self, key, code, default, pos): self.print_key_match(pos, key[pos:]) self.print(code, indent=4) self.print("} else {") self.print(default, indent=4) self.print("}") def print(self, *args, indent=0): self.file.write(" " * (self.indent + indent)) print(*args, file=self.file) def build_len_switch(self, keys, default): by_len = defaultdict(dict) for k, v in keys.items(): by_len[len(k)][k] = v with self.print_switch("len") as b1: for length, keys in sorted(by_len.items()): with b1.print_case(length, unquoted=True) as b2: b2.build_prefix_switch(keys, default, pos=0, length=length) b1.print("default:", default) def build_prefix_switch(self, keys, default, pos, length): if pos == length: for val in keys.values(): self.print(val) return by_prefix = defaultdict(dict) for k, v in keys.items(): by_prefix[k[pos]][k] = v if len(by_prefix) > 1: with self.print_switch("key[{}]".format(pos)) as b1: for prefix, keys in by_prefix.items(): with b1.print_case(prefix) as b2: b2.build_prefix_switch(keys, default, pos=pos + 1, length=length) b1.print("default:", default) else: # Only one key left, or many keys left with a common prefix if len(keys) == 1: key, val = next(iter(keys.items())) self.print_leaf_check(key, val, default, pos) else: self.build_common_prefix_switch(keys, default, pos, length) def build_common_prefix_switch(self, keys, default, pos, length): # Look for the common prefix prefix = os.path.commonprefix([x[pos:] for x in keys.keys()]) if len(prefix) == 1: self.print_key_match(pos, prefix) self.build_prefix_switch(keys, default, pos + 1, length) self.print("} else {") self.print(default, indent=self.indent_step) self.print("}") else: self.print_key_match(pos, prefix) with self.nest() as b: b.build_prefix_switch(keys, default, pos + len(prefix), length) self.print("} else {") self.print(default, indent=self.indent_step) self.print("}") class Table: re_case_line = re.compile(r'^\s*case\s*"(?P[^"]+)"\s*:\s*(?P.*)$') re_default_line = re.compile(r'^\s*default\s*:\s*(?P.+)$') re_trailing_break = re.compile(r"\s*break\s*;\s*$") def __init__(self, indent): self.keys = {} self.default = "" self.cur_key = None self.cur_code = None self.re_end_switch = re.compile("^" + re.escape(indent) + r"}\s*") self.indent = indent def _add_key_code(self): self.keys[self.cur_key] = self.re_trailing_break.sub("", self.cur_code) self.cur_key = None self.cur_code = None def _start_key(self, key, code): if self.cur_key is not None: self._add_key_code() self.cur_key = key self.cur_code = code def _add_code(self, code): if self.cur_code: self.cur_code += "\n" self.cur_code += code def _add_default(self, code): if self.cur_key: self._add_key_code() if self.default: self.default += "\n" self.default += code def _end_switch(self): if self.cur_key: self._add_key_code() def parse_line(self, lineno, line): if self.re_end_switch.match(line): self._end_switch() return Verbatim() mo = self.re_case_line.match(line) if mo: self._start_key(mo.group("key"), mo.group("code")) return None mo = self.re_default_line.match(line) if mo: self._add_default(mo.group("code")) return None self._add_code(line) return None def dump(self): for k, v in self.keys.items(): print("{}: {}".format(k, v)) def print(self, file): builder = Builder(len(self.indent), file=file) builder.build_len_switch(self.keys, self.default) class Verbatim: re_switch_line = re.compile(r"^(?P\s*)switch\s*\(key\)\s*{\s*//\s*mklookup\s*$") def __init__(self): self.lines = [] def parse_line(self, lineno, line): mo = self.re_switch_line.match(line) if not mo: self.lines.append(line) return None else: return Table(indent=mo.group("indent")) def print(self, file): for line in self.lines: print(line, file=file) def parse(fname): """ Parse a file into a sequence of blocks """ blocks = [Verbatim()] with open(fname) as fd: for lineno, line in enumerate(fd, start=1): next_block = blocks[-1].parse_line(lineno, line.rstrip()) if next_block is not None: blocks.append(next_block) return blocks @contextmanager def outfile(args): if args.outfile: with open(args.outfile, "wt", encoding="utf8") as fd: try: yield fd except Exception as e: if os.path.exists(args.outfile): os.unlink(args.outfile) raise e else: yield sys.stdout def main(): parser = argparse.ArgumentParser( description="build C code for a lookup table") parser.add_argument("--verbose", "-v", action="store_true", help="verbose output") parser.add_argument("--debug", action="store_true", help="debug output") parser.add_argument("-o", "--outfile", action="store", help="output file (default: stdout)") parser.add_argument("pathname", nargs="?", help="source file with the lookup table description") args = parser.parse_args() log_format = "%(asctime)-15s %(levelname)s %(message)s" level = logging.WARN if args.debug: level = logging.DEBUG elif args.verbose: level = logging.INFO logging.basicConfig(level=level, stream=sys.stderr, format=log_format) blocks = parse(args.pathname) with outfile(args) as out: for block in blocks: block.print(out) if __name__ == "__main__": try: main() except Fail as e: log.error("%s", e) sys.exit(1) dballe-8.6/dballe/Makefile.am0000644000175000017500000002256413554573614013040 00000000000000## Process this file with automake to produce Makefile.in EXTRA_DIST = CLEANFILES = noinst_DATA = AM_CPPFLAGS = -DTABLE_DIR=\"$(tabledir)\" -I$(top_srcdir) -I$(top_builddir) $(WREPORT_CFLAGS) $(LIBPQ_CFLAGS) $(SQLITE3_CFLAGS) $(MYSQL_CFLAGS) if FILE_OFFSET_BITS_64 AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64 endif common_libs = $(WREPORT_LIBS) $(LIBPQ_LIBS) $(SQLITE3_LIBS) $(MYSQL_LIBS) $(POPT_LIBS) # # Autobuilt files # core/query-access.cc: core/query-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ core/data-access.cc: core/data-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ core/shortcuts.h: vars.csv mkvars $(top_srcdir)/dballe/mkvars -t shortcuts.h $< -o $@ core/shortcuts.cc: vars.csv mkvars $(top_srcdir)/dballe/mkvars -t shortcuts.cc $< -o $@ core/shortcuts-access.in.cc: vars.csv mkvars $(top_srcdir)/dballe/mkvars -t shortcuts-access.in.cc $< -o $@ core/shortcuts-access.cc: core/shortcuts-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ msg/msg-extravars.h: vars.csv mkvars $(top_srcdir)/dballe/mkvars -t msg-extravars.h $< -o $@ msg/cursor-access.cc: msg/cursor-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ db/v7/cursor-access.cc: db/v7/cursor-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ db/summary-access.cc: db/summary-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ fortran/commonapi-access.cc: fortran/commonapi-access.in.cc mklookup $(top_srcdir)/dballe/mklookup $< -o $@ core/aliases.cc: core/aliases.gperf if ! gperf < $< > $@; then rm $@; /bin/false; fi BUILT_SOURCES = core/aliases.cc \ core/shortcuts.h core/shortcuts.cc core/shortcuts-access.in.cc core/shortcuts-access.cc msg/msg-extravars.h \ core/query-access.cc core/data-access.cc \ msg/cursor-access.cc db/v7/cursor-access.cc \ db/summary-access.cc fortran/commonapi-access.cc EXTRA_DIST += mklookup mkvars vars.csv \ core/aliases.gperf core/aliases.cc \ core/shortcuts.h core/shortcuts.cc core/shortcuts-access.in.cc core/shortcuts-access.cc msg/msg-extravars.h \ core/query-access.in.cc core/query-access.cc \ core/data-access.in.cc core/data-access.cc \ msg/cursor-access.in.cc msg/cursor-access.cc \ db/v7/cursor-access.in.cc db/v7/cursor-access.cc \ db/summary-access.in.cc db/summary-access.cc \ fortran/commonapi-access.in.cc fortran/commonapi-access.cc noinst_PROGRAMS = EXTRA_PROGRAMS = # # Shared library # dballeincludedir = $(includedir)/dballe nobase_dist_dballeinclude_HEADERS = \ fwd.h \ file.h \ importer.h \ exporter.h \ message.h \ types.h \ value.h \ values.h \ data.h \ query.h \ var.h \ cursor.h \ db.h \ python.h \ core/fwd.h \ core/error.h \ core/shortcuts.h \ core/enq.h \ core/cursor.h \ core/aliases.h \ core/benchmark.h \ core/defs.h \ core/var.h \ core/values.h \ core/file.h \ core/arrayfile.h \ core/csv.h \ core/data.h \ core/query.h \ core/structbuf.h \ core/matcher.h \ core/match-wreport.h \ core/smallset.h \ core/varmatch.h \ core/string.h \ core/trace.h \ core/json.h \ msg/fwd.h \ msg/bulletin.h \ msg/context.h \ msg/msg.h \ msg/msg-extravars.h \ msg/cursor.h \ msg/wr_codec.h \ msg/wr_exporters/common.h \ sql/fwd.h \ sql/sql.h \ sql/querybuf.h \ sql/sqlite.h \ db/fwd.h \ db/defs.h \ db/db.h \ db/v7/fwd.h \ db/v7/utils.h \ db/v7/trace.h \ db/v7/transaction.h \ db/v7/batch.h \ db/v7/cache.h \ db/v7/internals.h \ db/v7/repinfo.h \ db/v7/station.h \ db/v7/levtr.h \ db/v7/data.h \ db/v7/driver.h \ db/v7/sqlite/repinfo.h \ db/v7/sqlite/station.h \ db/v7/sqlite/levtr.h \ db/v7/sqlite/data.h \ db/v7/sqlite/driver.h \ db/v7/db.h \ db/v7/cursor.h \ db/v7/qbuilder.h \ db/summary.h \ db/explorer.h \ cmdline/cmdline.h \ cmdline/conversion.h \ cmdline/processor.h \ cmdline/dbadb.h \ fortran/api.h \ fortran/enq.h \ fortran/dbapi.h \ fortran/traced.h \ fortran/commonapi.h \ fortran/msgapi.h if HAVE_LIBPQ nobase_dist_dballeinclude_HEADERS += \ sql/postgresql.h \ db/v7/postgresql/repinfo.h \ db/v7/postgresql/station.h \ db/v7/postgresql/levtr.h \ db/v7/postgresql/data.h \ db/v7/postgresql/driver.h endif if HAVE_MYSQL nobase_dist_dballeinclude_HEADERS += \ sql/mysql.h \ db/v7/mysql/repinfo.h \ db/v7/mysql/station.h \ db/v7/mysql/levtr.h \ db/v7/mysql/data.h \ db/v7/mysql/driver.h endif lib_LTLIBRARIES = libdballe.la libdballe_la_SOURCES = \ file.cc \ importer.cc \ exporter.cc \ message.cc \ types.cc \ value.cc \ values.cc \ data.cc \ query.cc \ var.cc \ cursor.cc \ db.cc \ core/error.cc \ core/cursor.cc \ core/aliases.cc \ core/shortcuts.cc \ core/shortcuts-access.cc \ core/benchmark.cc \ core/defs.cc \ core/var.cc \ core/values.cc \ core/file.cc \ core/arrayfile.cc \ core/csv.cc \ core/data.cc \ core/data-access.cc \ core/query.cc \ core/query-access.cc \ core/structbuf.cc \ core/matcher.cc \ core/match-wreport.cc \ core/varmatch.cc \ core/json.cc \ core/string.cc \ msg/bulletin.cc \ msg/context.cc \ msg/msg.cc \ msg/cursor.cc \ msg/cursor-access.cc \ msg/wr_codec.cc \ msg/wr_importers/base.cc \ msg/wr_importers/synop.cc \ msg/wr_importers/ship.cc \ msg/wr_importers/temp.cc \ msg/wr_importers/metar.cc \ msg/wr_importers/flight.cc \ msg/wr_importers/generic.cc \ msg/wr_importers/pollution.cc \ msg/wr_exporters/common.cc \ msg/wr_exporters/synop.cc \ msg/wr_exporters/ship.cc \ msg/wr_exporters/buoy.cc \ msg/wr_exporters/metar.cc \ msg/wr_exporters/temp.cc \ msg/wr_exporters/flight.cc \ msg/wr_exporters/generic.cc \ msg/wr_exporters/pollution.cc \ sql/sql.cc \ sql/querybuf.cc \ sql/sqlite.cc \ db/db.cc \ db/v7/utils.cc \ db/v7/trace.cc \ db/v7/transaction.cc \ db/v7/batch.cc \ db/v7/cache.cc \ db/v7/repinfo.cc \ db/v7/station.cc \ db/v7/levtr.cc \ db/v7/data.cc \ db/v7/driver.cc \ db/v7/sqlite/repinfo.cc \ db/v7/sqlite/station.cc \ db/v7/sqlite/levtr.cc \ db/v7/sqlite/data.cc \ db/v7/sqlite/driver.cc \ db/v7/db.cc \ db/v7/cursor.cc \ db/v7/cursor-access.cc \ db/v7/qbuilder.cc \ db/v7/import.cc \ db/v7/export.cc \ db/summary.cc \ db/summary-access.cc \ db/explorer.cc \ cmdline/cmdline.cc \ cmdline/processor.cc \ cmdline/conversion.cc \ cmdline/dbadb.cc \ fortran/api.cc \ fortran/traced.cc \ fortran/commonapi.cc \ fortran/commonapi-access.cc \ fortran/msgapi.cc \ fortran/dbapi.cc if HAVE_LIBPQ libdballe_la_SOURCES += \ sql/postgresql.cc \ db/v7/postgresql/repinfo.cc \ db/v7/postgresql/station.cc \ db/v7/postgresql/levtr.cc \ db/v7/postgresql/data.cc \ db/v7/postgresql/driver.cc endif if HAVE_MYSQL libdballe_la_SOURCES += \ sql/mysql.cc \ db/v7/mysql/repinfo.cc \ db/v7/mysql/station.cc \ db/v7/mysql/levtr.cc \ db/v7/mysql/data.cc \ db/v7/mysql/driver.cc endif libdballe_la_CPPFLAGS = $(AM_CPPFLAGS) -Werror libdballe_la_LDFLAGS = -version-info @LIBDBALLE_VERSION_INFO@ $(common_libs) # filter.cc and sat.cc are here for now until I manage to turn them into a working system EXTRA_DIST += \ core/vasprintf.h \ core/byteswap.h \ msg/wr_importers/base.h # # Unit testing # check_PROGRAMS = test-dballe TESTS_ENVIRONMENT = $(top_srcdir)/extra/runtest #TESTS = $(check_PROGRAMS) check-local: $(TESTS_ENVIRONMENT) -v $(check_PROGRAMS) #CXXFLAGS += -O0 dist_noinst_HEADERS = \ core/tests.h \ msg/tests.h \ db/tests.h test_dballe_SOURCES = \ tests-main.cc \ core/tests.cc \ core/shortcuts-test.cc \ file-test.cc \ importer-test.cc \ exporter-test.cc \ message-test.cc \ types-test.cc \ value-test.cc \ values-test.cc \ data-test.cc \ query-test.cc \ var-test.cc \ cursor-test.cc \ db-test.cc \ core/cursor-test.cc \ core/aliases-test.cc \ core/defs-test.cc \ core/var-test.cc \ core/values-test.cc \ core/file-test.cc \ core/data-test.cc \ core/query-test.cc \ core/structbuf-test.cc \ core/csv-test.cc \ core/matcher-test.cc \ core/match-wreport-test.cc \ core/smallset-test.cc \ core/varmatch-test.cc \ core/json-test.cc \ core/string-test.cc \ msg/tests.cc \ msg/bulletin-test.cc \ msg/context-test.cc \ msg/msg-test.cc \ msg/cursor-test.cc \ msg/wr_codec_generic-test.cc \ msg/wr_codec-test.cc \ msg/wr_import-test.cc \ msg/wr_export-test.cc \ sql/querybuf-test.cc \ sql/sqlite-test.cc \ db/tests.cc \ db/v7/utils-test.cc \ db/v7/trace-test.cc \ db/v7/batch-test.cc \ db/v7/cache-test.cc \ db/v7/repinfo-test.cc \ db/v7/station-test.cc \ db/v7/levtr-test.cc \ db/v7/data-test.cc \ db/db-test.cc \ db/db-basic-test.cc \ db/db-misc-test.cc \ db/db-query-station-test.cc \ db/db-query-data-test.cc \ db/db-query-summary-test.cc \ db/db-import-test.cc \ db/db-export-test.cc \ db/summary-test.cc \ db/explorer-test.cc \ fortran/traced-test.cc \ fortran/commonapi-test.cc \ fortran/msgapi-test.cc \ fortran/dbapi-test.cc \ cmdline/processor-test.cc \ cmdline/dbadb-test.cc if HAVE_LIBPQ test_dballe_SOURCES += \ sql/postgresql-test.cc endif if HAVE_MYSQL test_dballe_SOURCES += \ sql/mysql-test.cc endif test_dballe_LDADD = \ libdballe.la \ $(common_libs) # # Benchmark # # TODO: remove/move code # noinst_PROGRAMS += bench-run # # dist_noinst_HEADERS += \ # db/benchmark.h # # bench_run_SOURCES = \ # db/db-query-parse-bench.cc \ # db/db-import-bench.cc \ # db/db-query-bench.cc # bench_run_LDADD = \ # libdballe.la \ # $(common_libs) # # Profile # noinst_PROGRAMS += profile-run profile_run_SOURCES = \ profile-main.cc profile_run_LDADD = \ libdballe.la \ $(common_libs) # toplevel EXTRA_DIST += msg/wr_exporters/exporters.dox \ msg/ltypes.txt dballe-8.6/dballe/types-test.cc0000644000175000017500000002764713554564112013434 00000000000000#include "core/tests.h" #include "types.h" using namespace std; using namespace wreport::tests; using namespace dballe; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("dballe_types"); void Tests::register_tests() { add_method("date", []() { wassert(actual(Date(2015, 1, 1)) == Date(2015, 1, 1)); wassert(actual(Date(2013, 1, 1)) < Date(2014, 1, 1)); wassert(actual(Date(2013, 1, 1)) < Date(2013, 2, 1)); wassert(actual(Date(2013, 1, 1)) < Date(2013, 1, 2)); wassert(actual(Date(1945, 4, 25)) != Date(1945, 4, 26)); }); add_method("time", []() { // Constructor boundary checks wassert(actual(Time( 0, 0, 0)) == Time(0, 0, 0)); wassert(actual(Time(23, 59, 60)) == Time(23, 59, 60)); wassert(actual(Time(13, 1, 1)) < Time(14, 1, 1)); wassert(actual(Time(13, 1, 1)) < Time(13, 2, 1)); wassert(actual(Time(13, 1, 1)) < Time(13, 1, 2)); wassert(actual(Time(19, 4, 25)) != Time(19, 4, 26)); }); add_method("datetime", []() { // Constructor boundary checks wassert(actual(Datetime(2015, 1, 1, 0, 0, 0)) == Datetime(2015, 1, 1, 0, 0, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2014, 1, 1, 0, 0, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2013, 2, 1, 0, 0, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2013, 1, 2, 0, 0, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2013, 1, 1, 1, 0, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2013, 1, 1, 0, 1, 0)); wassert(actual(Datetime(2013, 1, 1, 0, 0, 0)) < Datetime(2013, 1, 1, 0, 0, 1)); wassert(actual(Datetime(1945, 4, 25, 8, 0, 0)) != Datetime(1945, 4, 26, 8, 0, 0)); }); add_method("datetime_jdays", []() { // Test Date to/from julian days conversion Date d(2015, 4, 25); wassert(actual(d.to_julian()) == 2457138); d.from_julian(2457138); wassert(actual(d.year) == 2015); wassert(actual(d.month) == 4); wassert(actual(d.day) == 25); }); add_method("datetimerange", []() { Datetime missing; Datetime dt_2010(2010, 1, 1, 0, 0, 0); Datetime dt_2011(2011, 1, 1, 0, 0, 0); Datetime dt_2012(2012, 1, 1, 0, 0, 0); Datetime dt_2013(2013, 1, 1, 0, 0, 0); // Test equality wassert(actual(DatetimeRange(missing, missing) == DatetimeRange(missing, missing)).istrue()); wassert(actual(DatetimeRange(dt_2010, dt_2011) == DatetimeRange(dt_2010, dt_2011)).istrue()); wassert(actual(DatetimeRange(dt_2010, missing) == DatetimeRange(missing, missing)).isfalse()); wassert(actual(DatetimeRange(missing, dt_2010) == DatetimeRange(missing, missing)).isfalse()); wassert(actual(DatetimeRange(missing, missing) == DatetimeRange(dt_2010, missing)).isfalse()); wassert(actual(DatetimeRange(missing, missing) == DatetimeRange(missing, dt_2010)).isfalse()); wassert(actual(DatetimeRange(dt_2010, dt_2011) == DatetimeRange(dt_2012, dt_2013)).isfalse()); // Test contains wassert(actual(DatetimeRange(missing, missing).contains(DatetimeRange(missing, missing))).istrue()); wassert(actual(DatetimeRange(dt_2010, dt_2011).contains(DatetimeRange(dt_2010, dt_2011))).istrue()); wassert(actual(DatetimeRange(missing, missing).contains(DatetimeRange(dt_2011, dt_2012))).istrue()); wassert(actual(DatetimeRange(dt_2011, dt_2012).contains(DatetimeRange(missing, missing))).isfalse()); wassert(actual(DatetimeRange(dt_2010, dt_2013).contains(DatetimeRange(dt_2011, dt_2012))).istrue()); wassert(actual(DatetimeRange(dt_2010, dt_2012).contains(DatetimeRange(dt_2011, dt_2013))).isfalse()); wassert(actual(DatetimeRange(missing, dt_2010).contains(DatetimeRange(dt_2011, missing))).isfalse()); }); add_method("coords", []() { wassert(actual(Coords(44.0, 11.0)) == Coords(44.0, 360.0+11.0)); }); add_method("latrange", []() { double dmin, dmax; LatRange lr; wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == LatRange::IMIN); wassert(actual(lr.imax) == LatRange::IMAX); wassert(actual(lr.dmin()) == LatRange::DMIN); wassert(actual(lr.dmax()) == LatRange::DMAX); lr.get(dmin, dmax); wassert(actual(dmin) == LatRange::DMIN); wassert(actual(dmax) == LatRange::DMAX); wassert(actual(lr.contains(0)).istrue()); lr = LatRange(40.0, 50.0); wassert(actual(lr.is_missing()).isfalse()); wassert(actual(lr.imin) == 4000000); wassert(actual(lr.imax) == 5000000); lr.get(dmin, dmax); wassert(actual(dmin) == 40.0); wassert(actual(dmax) == 50.0); wassert(actual(lr.contains(39.9)).isfalse()); wassert(actual(lr.contains(40.0)).istrue()); wassert(actual(lr.contains(45.0)).istrue()); wassert(actual(lr.contains(50.0)).istrue()); wassert(actual(lr.contains(50.1)).isfalse()); wassert(actual(lr.contains(4500000)).istrue()); wassert(actual(lr.contains(5500000)).isfalse()); lr.set(-10.0, 10.0); wassert(actual(lr.imin) == -1000000); wassert(actual(lr.imax) == 1000000); wassert(actual(lr) == LatRange(-10.0, 10.0)); lr.set(4000000, 5000000); wassert(actual(lr.imin) == 4000000); wassert(actual(lr.imax) == 5000000); wassert(actual(lr) == LatRange(40.0, 50.0)); }); add_method("lonrange", []() { double dmin, dmax; LonRange lr; wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == MISSING_INT); wassert(actual(lr.imax) == MISSING_INT); wassert(actual(lr.dmin()) == -180.0); wassert(actual(lr.dmax()) == 180.0); lr.get(dmin, dmax); wassert(actual(dmin) == -180.0); wassert(actual(dmax) == 180.0); wassert(actual(lr.contains(0)).istrue()); lr = LonRange(-18000000, 18000000); wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == MISSING_INT); wassert(actual(lr.imax) == MISSING_INT); lr = LonRange(-180.0, 180.0); wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == MISSING_INT); wassert(actual(lr.imax) == MISSING_INT); lr.set(-18000000, 18000000); wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == MISSING_INT); wassert(actual(lr.imax) == MISSING_INT); lr.set(-180.0, 180.0); wassert(actual(lr.is_missing()).istrue()); wassert(actual(lr.imin) == MISSING_INT); wassert(actual(lr.imax) == MISSING_INT); lr = LonRange(40.0, 50.0); wassert(actual(lr.is_missing()).isfalse()); wassert(actual(lr.imin) == 4000000); wassert(actual(lr.imax) == 5000000); lr.get(dmin, dmax); wassert(actual(dmin) == 40.0); wassert(actual(dmax) == 50.0); wassert(actual(lr.contains(39.9)).isfalse()); wassert(actual(lr.contains(40.0)).istrue()); wassert(actual(lr.contains(45.0)).istrue()); wassert(actual(lr.contains(50.0)).istrue()); wassert(actual(lr.contains(50.1)).isfalse()); wassert(actual(lr.contains(4500000)).istrue()); wassert(actual(lr.contains(5500000)).isfalse()); lr = LonRange(50.0, 40.0); wassert(actual(lr.is_missing()).isfalse()); wassert(actual(lr.imin) == 5000000); wassert(actual(lr.imax) == 4000000); lr.get(dmin, dmax); wassert(actual(dmin) == 50.0); wassert(actual(dmax) == 40.0); wassert(actual(lr.contains(39.9)).istrue()); wassert(actual(lr.contains(40.0)).istrue()); wassert(actual(lr.contains(45.0)).isfalse()); wassert(actual(lr.contains(50.0)).istrue()); wassert(actual(lr.contains(50.1)).istrue()); wassert(actual(lr.contains(4500000)).isfalse()); wassert(actual(lr.contains(5500000)).istrue()); lr.set(-10.0, 10.0); wassert(actual(lr.imin) == -1000000); wassert(actual(lr.imax) == 1000000); wassert(actual(lr) == LonRange(-10.0, 10.0)); lr.set(350.0, 360.0); wassert(actual(lr.imin) == -1000000); wassert(actual(lr.imax) == 0); wassert(actual(lr) == LonRange(-10.0, 0.0)); lr.set(0.0, 360.0); //wassert(actual(lr.imin) == -1000000); //wassert(actual(lr.imax) == 0); //wassert(actual(lr) == LonRange(-10.0, 0.0)); wassert_true(lr.contains(180.0)); lr = LonRange(); lr.imin = 1000000; wassert_true(lr.is_missing()); wassert(actual(lr) == LonRange()); }); add_method("level_descs", []() { // Try to get descriptions for all the layers for (int i = 0; i < 261; ++i) { Level(i).describe(); Level(i, 0).describe(); Level(i, MISSING_INT, i, MISSING_INT).describe(); Level(i, 0, i, 0).describe(); } }); add_method("trange_descs", []() { // Try to get descriptions for all the time ranges for (int i = 0; i < 256; ++i) { Trange(i).describe(); Trange(i, 0).describe(); Trange(i, 0, 0).describe(); } }); add_method("known_descs", []() { // Verify some well-known descriptions wassert(actual(Level().describe()) == "Information about the station that generated the data"); wassert(actual(Level(103, 2000).describe()) == "2.000m above ground"); wassert(actual(Level(103, 2000, 103, 4000).describe()) == "Layer from [2.000m above ground] to [4.000m above ground]"); wassert(actual(Trange(254, 86400).describe()) == "Forecast at t+1d, instantaneous value"); wassert(actual(Trange(2, 0, 43200).describe()) == "Maximum over 12h at forecast time 0"); wassert(actual(Trange(3, 194400, 43200).describe()) == "Minimum over 12h at forecast time 2d 6h"); }); add_method("datetime_bounds", []() { #define MISSING_DTOS MISSING_INT, MISSING_INT, MISSING_INT, MISSING_INT wassert(actual(Datetime::lower_bound(2000, MISSING_INT, MISSING_DTOS)) == Datetime(2000, 1, 1, 0, 0, 0)); wassert(actual(Datetime::upper_bound(2000, 2, MISSING_DTOS)) == Datetime(2000, 2, 29, 23, 59, 59)); wassert(actual(Datetime::upper_bound(2002, 2, MISSING_DTOS)) == Datetime(2002, 2, 28, 23, 59, 59)); wassert(actual(Datetime::upper_bound(2004, 2, MISSING_DTOS)) == Datetime(2004, 2, 29, 23, 59, 59)); auto e = wassert_throws(wreport::error_consistency, Datetime::lower_bound(MISSING_INT, 2, MISSING_DTOS)); wassert(actual(e.what()) == "month 2 given with no year"); #undef MISSING_DTOS }); add_method("ident", []() { wassert(actual(Ident()) == Ident()); wassert(actual(Ident("foo")) == Ident("foo")); wassert(actual(Ident().is_missing()).istrue()); wassert(actual(Ident("foo").is_missing()).isfalse()); Ident test; wassert(actual((const char*)test == nullptr).istrue()); test = "antani"; wassert(actual((const char*)test) == "antani"); Ident test1 = test; wassert(actual(test) == Ident("antani")); wassert(actual(test1) == Ident("antani")); test1 = test1; wassert(actual(test) == Ident("antani")); wassert(actual(test1) == Ident("antani")); test1 = Ident("blinda"); wassert(actual(test) == Ident("antani")); wassert(actual(test1) == Ident("blinda")); test = move(test1); wassert(actual(test) == Ident("blinda")); wassert(actual(test1.is_missing()).istrue()); Ident test2(move(test)); wassert(actual(test.is_missing()).istrue()); wassert(actual(test2) == Ident("blinda")); test = string("foo"); wassert(actual(test) == Ident("foo")); wassert(actual(test2) == Ident("blinda")); wassert(actual(Ident("foo") == Ident("foo")).istrue()); wassert(actual(Ident("foo") <= Ident("foo")).istrue()); wassert(actual(Ident("foo") >= Ident("foo")).istrue()); wassert(actual(Ident("foo") == Ident("bar")).isfalse()); wassert(actual(Ident("foo") != Ident("foo")).isfalse()); wassert(actual(Ident("foo") != Ident("bar")).istrue()); wassert(actual(Ident("antani") < Ident("blinda")).istrue()); wassert(actual(Ident("antani") <= Ident("blinda")).istrue()); wassert(actual(Ident("antani") > Ident("blinda")).isfalse()); wassert(actual(Ident("antani") >= Ident("blinda")).isfalse()); wassert(actual(Ident("blinda") < Ident("antani")).isfalse()); wassert(actual(Ident("blinda") <= Ident("antani")).isfalse()); wassert(actual(Ident("blinda") > Ident("antani")).istrue()); wassert(actual(Ident("blinda") >= Ident("antani")).istrue()); }); } } dballe-8.6/dballe/cmdline/0000755000175000017500000000000013602152021012441 500000000000000dballe-8.6/dballe/cmdline/conversion.cc0000644000175000017500000002130413554564112015073 00000000000000#include "conversion.h" #include "processor.h" #include "dballe/file.h" #include "dballe/message.h" #include "dballe/exporter.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include using namespace wreport; using namespace std; namespace dballe { namespace cmdline { Converter::~Converter() { if (file) delete file; if (exporter) delete exporter; } void Converter::process_bufrex_msg(const BinaryMessage& orig, const Bulletin& msg) { string raw; try { raw = msg.encode(); } catch (std::exception& e) { throw ProcessingException(orig.pathname, orig.index, e); } file->write(raw); } void Converter::process_dba_msg(const BinaryMessage& orig, const std::vector>& msgs) { string raw; try { raw = exporter->to_binary(msgs); } catch (std::exception& e) { throw ProcessingException(orig.pathname, orig.index, e); } file->write(raw); } // Recompute data_category and data_subcategory according to WMO international values static void compute_wmo_categories(Bulletin& b, const Bulletin& orig, const std::vector>& msgs) { b.data_category = orig.data_category; b.data_subcategory_local = 255; switch (orig.data_category) { case 0: { // BC01-SYNOP // Get the hour from the first message // Default to 1 to simulate an odd observation time int hour = msgs[0]->get_datetime().is_missing() ? 1 : msgs[0]->get_datetime().hour; if ((hour % 6) == 0) // 002 at main synoptic times 00, 06, 12, 18 UTC, b.data_subcategory = 2; else if ((hour % 3 == 0)) // 001 at intermediate synoptic times 03, 09, 15, 21 UTC, b.data_subcategory = 1; else // 000 at observation times 01, 02, 04, 05, 07, 08, 10, 11, 13, 14, 16, 17, 19, 20, 22 and 23 UTC. b.data_subcategory = 0; break; } case 1: // BC10-SHIP // If required, the international data sub-category shall be included for SHIP data as 000 at all // observation times 00, 01, 02, ..., 23 UTC. b.data_subcategory = 0; break; case 2: // BC20-PILOT // BC25-TEMP switch (msgs[0]->get_type()) { // 001 for PILOT data, case MessageType::PILOT: b.data_subcategory = 1; // ncdf_pilot = 4 ,& ! indicator for proc. NetCDF PILOT (z-levels) input // ncdf_pilot_p = 5 ,& ! indicator for proc. NetCDF PILOT (p-levels) input break; // 002 for PILOT SHIP data, (TODO) // 003 for PILOT MOBIL data. (TODO) // 004 for TEMP data, case MessageType::TEMP: b.data_subcategory = 4; break; // 005 for TEMP SHIP data, case MessageType::TEMP_SHIP: b.data_subcategory = 5; break; // 006 for TEMP MOBIL data (TODO) // Default to TEMP default: b.data_subcategory = 4; break; // TODO-items are not supported since I have never seen one } break; // Missing data from this onwards case 3: b.data_subcategory = 0; break; case 4: switch (msgs[0]->get_type()) { case MessageType::AIREP: b.data_subcategory = 1; break; default: b.data_subcategory = 0; break; } break; case 5: b.data_subcategory = 0; break; case 6: b.data_subcategory = 0; break; case 7: b.data_subcategory = 0; break; case 8: b.data_subcategory = 0; break; case 9: b.data_subcategory = 0; break; case 10: b.data_subcategory = 1; break; case 12: b.data_subcategory = 0; break; case 21: b.data_subcategory = 5; break; case 31: b.data_subcategory = 0; break; case 101: b.data_subcategory = 7; break; default: b.data_subcategory = 255; break; } } // Compute local data_subcategory to tell bufr2netcdf output files apart using // lokal-specific categorisation static void compute_bufr2netcdf_categories(Bulletin& b, const Bulletin& orig, const std::vector>& msgs) { switch (orig.data_category) { case 0: // Force data_subcategory to 0, as bufr2netcdf processing doesn't need the // hour distinction b.data_subcategory = 0; // 13 for fixed stations // 14 for mobile stations b.data_subcategory_local = 13; if (const wreport::Var* v = impl::Message::downcast(msgs[0])->get_ident_var()) if (v->isset()) b.data_subcategory_local = 14; break; case 2: if (b.data_subcategory == 1) { // 4 for z-level pilots // 5 for p-level pilots // Arbitrary default to z-level pilots auto msg = impl::Message::downcast(msgs[0]); b.data_subcategory_local = 4; for (const auto& ctx: msg->data) { switch (ctx.level.ltype1) { case 100: // Isobaric Surface b.data_subcategory_local = 5; break; case 102: // Specific Altitude Above Mean Sea Level b.data_subcategory_local = 4; break; } } } break; case 4: switch (msgs[0]->get_type()) { case MessageType::AMDAR: b.data_subcategory_local = 8; break; case MessageType::ACARS: b.data_subcategory_local = 9; break; default: break; } break; } } void Converter::process_dba_msg_from_bulletin(const BinaryMessage& orig, const Bulletin& bulletin, const std::vector>& msgs) { string raw; try { unique_ptr b1 = exporter->to_bulletin(msgs); if (bufr2netcdf_categories) { compute_wmo_categories(*b1, bulletin, msgs); compute_bufr2netcdf_categories(*b1, bulletin, msgs); } else { b1->data_category = bulletin.data_category; b1->data_subcategory = bulletin.data_subcategory; b1->data_subcategory_local = bulletin.data_subcategory_local; } raw = b1->encode(); } catch (std::exception& e) { throw ProcessingException(orig.pathname, orig.index, e); } file->write(raw); } bool Converter::operator()(const cmdline::Item& item) { if (item.msgs == NULL || item.msgs->size() == 0) { fprintf(stderr, "No interpreted information available: is a recoding enough?\n"); // See if we can just recode the raw data // We want bufrex raw data if (item.bulletin == NULL) { fprintf(stderr, "No BUFREX raw data to attempt low-level bufrex recoding\n"); return false; } // No report override if (dest_rep_memo != NULL) { fprintf(stderr, "report override not allowed for low-level bufrex recoding\n"); return false; } // No template change if (dest_template != NULL) { fprintf(stderr, "template change not supported for low-level bufrex recoding\n"); return false; } // Same encoding if ((file->encoding() == Encoding::BUFR && string(item.bulletin->encoding_name()) == "CREX") || (file->encoding() == Encoding::CREX && string(item.bulletin->encoding_name()) == "BUFR")) { fprintf(stderr, "encoding change not yet supported for low-level bufrex recoding\n"); return false; } // We can just recode the raw braw fprintf(stderr, "we can do a low-level bufrex recoding\n"); process_bufrex_msg(*item.rmsg, *item.bulletin); return true; } if (dest_rep_memo != NULL) { // Force message type (will also influence choice of template later) MessageType type = impl::Message::type_from_repmemo(dest_rep_memo); for (auto& msg: *item.msgs) impl::Message::downcast(msg)->type = type; } if (item.bulletin and dest_rep_memo == NULL) process_dba_msg_from_bulletin(*item.rmsg, *item.bulletin, *item.msgs); else process_dba_msg(*item.rmsg, *item.msgs); return true; } } } dballe-8.6/dballe/cmdline/processor-test.cc0000644000175000017500000001223413554564112015704 00000000000000#include "dballe/core/tests.h" #include "processor.h" #include using namespace dballe; using namespace dballe::cmdline; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("cmdline_processor"); void Tests::register_tests() { add_method("parse_index", [] { Filter filter; filter.set_index_filter("99-101"); wassert(actual(filter.match_index(98)).isfalse()); wassert(actual(filter.match_index(99)).istrue()); wassert(actual(filter.match_index(100)).istrue()); wassert(actual(filter.match_index(101)).istrue()); wassert(actual(filter.match_index(102)).isfalse()); filter.set_index_filter("100-101"); wassert(actual(filter.match_index(99)).isfalse()); wassert(actual(filter.match_index(100)).istrue()); wassert(actual(filter.match_index(101)).istrue()); wassert(actual(filter.match_index(102)).isfalse()); filter.set_index_filter("-10, 100-101, 103, 105-"); wassert(actual(filter.imatcher.ranges.size()) == 4u); wassert(actual(filter.imatcher.ranges[0].first) == 0); wassert(actual(filter.imatcher.ranges[0].second) == 10); wassert(actual(filter.imatcher.ranges[1].first) == 100); wassert(actual(filter.imatcher.ranges[1].second) == 101); wassert(actual(filter.imatcher.ranges[2].first) == 103); wassert(actual(filter.imatcher.ranges[2].second) == 103); wassert(actual(filter.imatcher.ranges[3].first) == 105); wassert(actual(filter.imatcher.ranges[3].second) == std::numeric_limits::max()); wassert(actual(filter.match_index(0)).istrue()); wassert(actual(filter.match_index(10)).istrue()); wassert(actual(filter.match_index(11)).isfalse()); wassert(actual(filter.match_index(99)).isfalse()); wassert(actual(filter.match_index(100)).istrue()); wassert(actual(filter.match_index(101)).istrue()); wassert(actual(filter.match_index(102)).isfalse()); wassert(actual(filter.match_index(103)).istrue()); wassert(actual(filter.match_index(104)).isfalse()); wassert(actual(filter.match_index(105)).istrue()); wassert(actual(filter.match_index(100000)).istrue()); filter.set_index_filter(""); wassert(actual(filter.match_index(0)).istrue()); wassert(actual(filter.match_index(10)).istrue()); }); add_method("parse_json", [] { struct TestAction : public Action { std::vector> messages; virtual bool operator()(const Item& item) { for (auto& m: *(item.msgs)) { messages.push_back(m->clone()); } return true; } }; ReaderOptions opts; opts.input_type = "json"; Reader reader(opts); TestAction action; reader.read({dballe::tests::datafile("/json/issue134.json")}, action); wassert(actual(action.messages.size()) == 5); { const wreport::Var* var = action.messages.at(0)->get(dballe::Level(103, 2000), dballe::Trange(254, 0, 0), WR_VAR(0, 12, 101)); wassert(actual(var) != (const wreport::Var*)0); const wreport::Var* attr = var->enqa(WR_VAR(0, 33, 7)); wassert(actual(attr) != (const wreport::Var*)0); wassert(actual(attr->enqi()) == 0); } { const wreport::Var* var = action.messages.at(1)->get(dballe::Level(103, 2000), dballe::Trange(254, 0, 0), WR_VAR(0, 12, 101)); wassert(actual(var) != (const wreport::Var*)0); const wreport::Var* attr = var->enqa(WR_VAR(0, 33, 7)); wassert(actual(attr) == (const wreport::Var*)0); } { const wreport::Var* var = action.messages.at(2)->get(dballe::Level(103, 2000), dballe::Trange(254, 0, 0), WR_VAR(0, 12, 101)); wassert(actual(var) != (const wreport::Var*)0); const wreport::Var* attr = var->enqa(WR_VAR(0, 8, 44)); wassert(actual(attr) != (const wreport::Var*)0); wassert(actual(attr->enqs()) == "ciao"); } { const wreport::Var* var = action.messages.at(3)->get(dballe::Level(103, 2000), dballe::Trange(254, 0, 0), WR_VAR(0, 12, 101)); wassert(actual(var) != (const wreport::Var*)0); const wreport::Var* attr = var->enqa(WR_VAR(0, 12, 102)); wassert(actual(attr) != (const wreport::Var*)0); wassert(actual(attr->enqd()) == 0.1); } { const wreport::Var* var = action.messages.at(4)->get(dballe::Level(103, 2000), dballe::Trange(254, 0, 0), WR_VAR(0, 12, 101)); wassert(actual(var) != (const wreport::Var*)0); const wreport::Var* attr1 = var->enqa(WR_VAR(0, 8, 44)); wassert(actual(attr1) != (const wreport::Var*)0); wassert(actual(attr1->enqs()) == "ciao"); const wreport::Var* attr2 = var->enqa(WR_VAR(0, 12, 102)); wassert(actual(attr2) != (const wreport::Var*)0); wassert(actual(attr2->enqd()) == 0.1); } }); add_method("issue77", [] { struct TestAction : public Action { virtual bool operator()(const Item& item) { return true; } }; ReaderOptions opts; opts.input_type = "json"; Reader reader(opts); TestAction action; reader.read({dballe::tests::datafile("/json/issue77.json")}, action); }); } } dballe-8.6/dballe/cmdline/dbadb.h0000644000175000017500000000224413554564112013606 00000000000000#ifndef DBALLE_CMDLINE_DBADB_H #define DBALLE_CMDLINE_DBADB_H #include #include #include #include #include #include namespace dballe { namespace cmdline { class Dbadb { protected: DB& db; public: Dbadb(DB& db) : db(db) {} /// Query data in the database and output results as arbitrary human readable text int do_dump(const Query& query, FILE* out); /// Query stations in the database and output results as arbitrary human readable text int do_stations(const Query& query, FILE* out); /// Export messages and dump their contents to the given file descriptor int do_export_dump(const Query& query, FILE* out); /// Import the given files int do_import(const std::list& fnames, Reader& reader, const DBImportOptions& opts); /// Import one file int do_import(const std::string& fname, Reader& reader, const DBImportOptions& opts); /// Export messages writing them to the givne file int do_export(const Query& query, File& file, const char* output_template=NULL, const char* forced_repmemo=NULL); }; } } #endif dballe-8.6/dballe/cmdline/conversion.h0000644000175000017500000000307113554564112014736 00000000000000#ifndef DBALLE_CMDLINE_CONVERSION_H #define DBALLE_CMDLINE_CONVERSION_H #include namespace wreport { struct Bulletin; } namespace dballe { struct File; namespace cmdline { struct Converter : public Action { File* file; const char* dest_rep_memo; const char* dest_template; bool bufr2netcdf_categories; Exporter* exporter; Converter() : file(0), dest_rep_memo(0), dest_template(0), bufr2netcdf_categories(false), exporter(0) {} ~Converter(); /** * Convert the item as configured in the Converter, and write it to the * output file */ virtual bool operator()(const cmdline::Item& item); protected: /** * Perform conversion at the encoding level only (e.g. BUFR->CREX) * * @param orig * Original BinaryMessage used for its source information, to report errors * @param msg * Decoded wreport::Bulletin to to convert */ void process_bufrex_msg(const BinaryMessage& orig, const wreport::Bulletin& msg); /** * Perform conversion of decoded data, auto-inferring * type/subtype/localsubtype from the Messages contents */ void process_dba_msg(const BinaryMessage& orig, const std::vector>& msgs); /** * Perform conversion of decded data, using the original bulletin for * type/subtype/localsubtype information */ void process_dba_msg_from_bulletin(const BinaryMessage& orig, const wreport::Bulletin& bulletin, const std::vector>& msgs); }; } } #endif dballe-8.6/dballe/cmdline/processor.h0000644000175000017500000001252713554564112014576 00000000000000#ifndef DBALLE_CMDLINE_PROCESSOR_H #define DBALLE_CMDLINE_PROCESSOR_H #include #include #include #include #include #include #define DBALLE_JSON_VERSION "0.1" namespace wreport { struct Bulletin; } namespace dballe { namespace cmdline { /** * Exception used to embed processing issues that mean that processing of the * current element can safely be skipped. * * When this exception is caught we know, for example, that no output has been * produced for the item currently being processed. */ struct ProcessingException : public std::exception { std::string filename; unsigned index; std::string msg; /** * Create a new exception * * @param filename Input file being processed * @param index Index of the data being processed in the input file * @param msg Error message */ ProcessingException( const std::string& filename, unsigned index, const std::string& msg) : filename(filename), index(index) { initmsg(filename, index, msg.c_str()); } /** * Create a new exception * * @param filename Input file being processed * @param index Index of the data being processed in the input file * @param original (optional) original exception that was caught from the * underlying subsystem */ ProcessingException( const std::string& filename, unsigned index, const std::exception& original) : filename(filename), index(index) { initmsg(filename, index, original.what()); } /** * Create a new exception * * @param filename Input file being processed * @param index Index of the data being processed in the input file * @param msg Error message * @param original (optional) original exception that was caught from the * underlying subsystem */ ProcessingException( const std::string& filename, unsigned index, const std::string& msg, const std::exception& original) : filename(filename), index(index) { initmsg(filename, index, msg.c_str()); this->msg += ": "; this->msg += original.what(); } virtual ~ProcessingException() throw() {} virtual const char* what() const throw () { return msg.c_str(); } protected: void initmsg(const std::string& fname, unsigned index, const char* msg); }; struct Item { unsigned idx; BinaryMessage* rmsg; wreport::Bulletin* bulletin; std::vector>* msgs; Item(); ~Item(); /// Decode all that can be decoded void decode(Importer& imp, bool print_errors=false); /// Set the value of msgs, possibly replacing the previous one void set_msgs(std::vector>* new_msgs); /// Throw a ProcessingException based on e void processing_failed(std::exception& e) const __attribute__ ((noreturn)); }; struct Action { virtual ~Action() {} virtual bool operator()(const Item& item) = 0; }; struct IndexMatcher { std::vector> ranges; void parse(const std::string& str); bool match(int val) const; }; struct ReaderOptions { int category = -1; int subcategory = -1; int checkdigit = -1; int unparsable = 0; int parsable = 0; const char* index_filter = nullptr; const char* input_type = "auto"; const char* fail_file_name = nullptr; }; struct Filter { impl::ExporterOptions export_opts; int category = -1; int subcategory = -1; int checkdigit = -1; int unparsable = 0; int parsable = 0; IndexMatcher imatcher; Matcher* matcher = nullptr; Filter(); Filter(const ReaderOptions& opts); ~Filter(); void set_index_filter(const std::string& val); /// Reset to the empty matcher void matcher_reset(); /// Initialise the matcher from a record void matcher_from_record(const Query& query); bool match_index(int idx) const; bool match_common(const BinaryMessage& rmsg, const std::vector>* msgs) const; bool match_msgs(const std::vector>& msgs) const; bool match_bufrex(const BinaryMessage& rmsg, const wreport::Bulletin* rm, const std::vector>* msgs) const; bool match_bufr(const BinaryMessage& rmsg, const wreport::Bulletin* rm, const std::vector>* msgs) const; bool match_crex(const BinaryMessage& rmsg, const wreport::Bulletin* rm, const std::vector>* msgs) const; bool match_item(const Item& item) const; }; class Reader { protected: std::string input_type; const char* fail_file_name; void read_csv(const std::list& fnames, Action& action); void read_json(const std::list& fnames, Action& action); void read_file(const std::list& fnames, Action& action); public: impl::ImporterOptions import_opts; Filter filter; bool verbose = false; unsigned count_successes = 0; unsigned count_failures = 0; Reader(const ReaderOptions& opts); bool has_fail_file() const; void read(const std::list& fnames, Action& action); }; } } #endif dballe-8.6/dballe/cmdline/dbadb.cc0000644000175000017500000001070613554564112013746 00000000000000#include "dbadb.h" #include "dballe/message.h" #include "dballe/msg/msg.h" #include "dballe/values.h" #include "dballe/db/db.h" #include using namespace wreport; using namespace std; // extern int op_verbose; namespace dballe { namespace cmdline { namespace { struct Importer : public Action { dballe::DB& db; const DBImportOptions& opts; std::shared_ptr transaction; Importer(dballe::DB& db, const DBImportOptions& opts) : db(db), opts(opts) {} virtual bool operator()(const cmdline::Item& item); void commit() { if (transaction.get()) transaction->commit(); } }; bool Importer::operator()(const Item& item) { if (!transaction.get()) transaction = db.transaction(); if (item.msgs == NULL) { fprintf(stderr, "Message #%d cannot be parsed: ignored\n", item.idx); return false; } try { transaction->import_messages(*item.msgs, opts); } catch (std::exception& e) { item.processing_failed(e); } return true; } } /// Query data in the database and output results as arbitrary human readable text int Dbadb::do_dump(const Query& query, FILE* out) { auto tr = db.transaction(); auto cursor = tr->query_data(query); for (unsigned i = 0; cursor->next(); ++i) { fprintf(out, "#%u: -----------------------\n", i); fprintf(out, "Station: "); cursor->get_station().print(out); fprintf(out, "Datetime: "); cursor->get_datetime().print(out); fprintf(out, "Level: "); cursor->get_level().print(out); fprintf(out, "Trange: "); cursor->get_trange().print(out); fprintf(out, "Var: "); cursor->get_var().print(out); } tr->rollback(); return 0; } /// Query stations in the database and output results as arbitrary human readable text int Dbadb::do_stations(const Query& query, FILE* out) { auto tr = db.transaction(); auto cursor = tr->query_stations(query); for (unsigned i = 0; cursor->next(); ++i) { fprintf(out, "#%u: -----------------------\n", i); fprintf(out, "Station: "); cursor->get_station().print(out); auto values = cursor->get_values(); for (const auto& val: values) { fprintf(out, "Var: "); val->print(out); } } tr->rollback(); return 0; } int Dbadb::do_export_dump(const Query& query, FILE* out) { auto cursor = db.query_messages(query); while (cursor->next()) cursor->get_message().print(out); return 0; } int Dbadb::do_import(const list& fnames, Reader& reader, const DBImportOptions& opts) { Importer importer(db, opts); reader.read(fnames, importer); importer.commit(); if (reader.verbose) fprintf(stderr, "%u messages successfully imported, %u messages skipped\n", reader.count_successes, reader.count_failures); // As discussed in #101, if there are both successes and failures, return // success only if --rejected has been used, because in that case the // caller can check the size of the rejected file to detect the difference // between a mixed result and a complete success. // One can use --rejected=/dev/null to ignore partial failures. if (!reader.count_failures) return 0; if (!reader.count_successes) return 1; return reader.has_fail_file() ? 0 : 1; } int Dbadb::do_import(const std::string& fname, Reader& reader, const DBImportOptions& opts) { list fnames; fnames.push_back(fname); return do_import(fnames, reader, opts); } int Dbadb::do_export(const Query& query, File& file, const char* output_template, const char* forced_repmemo) { impl::ExporterOptions opts; if (output_template && output_template[0] != 0) opts.template_name = output_template; if (forced_repmemo) forced_repmemo = forced_repmemo; auto exporter = Exporter::create(file.encoding(), opts); auto cursor = db.query_messages(query); while (cursor->next()) { auto msg = cursor->detach_message(); /* Override the message type if the user asks for it */ if (forced_repmemo != NULL) { impl::Message& m = impl::Message::downcast(*msg); m.type = impl::Message::type_from_repmemo(forced_repmemo); m.set_rep_memo(forced_repmemo); } std::vector> msgs; msgs.emplace_back(move(msg)); file.write(exporter->to_binary(msgs)); } return 0; } } } dballe-8.6/dballe/cmdline/cmdline.cc0000644000175000017500000003533313572421562014331 00000000000000#include "cmdline.h" #include "dballe/core/query.h" #include "dballe/msg/wr_codec.h" #include #include #include #include #include #include #include #include using namespace std; namespace dballe { namespace cmdline { static int is_tableend(struct poptOption* op) { return op->longName == NULL && \ op->shortName == '\0' && \ op->argInfo == 0 && \ op->arg == 0 && \ op->val == 0 && \ op->descrip == NULL && \ op->argDescrip == NULL; } static void manpage_print_options(const char* title, struct poptOption* optable, FILE* out) { int i; fprintf(out, ".SS %s\n", title); for (i = 0; !is_tableend(&(optable[i])); i++) { int bol = 1; if (optable[i].argInfo == POPT_ARG_INCLUDE_TABLE) continue; /* .TP .B \-d, \-\-distance (only valid for the \fIrelated\fP command) Set the maximum distance to use for the "related" option. (see the \fI\-\-distance\fP option in \fBtagcoll\fP(1)) */ fprintf(out, ".TP\n.B"); if (optable[i].argInfo == POPT_ARG_NONE) { if (optable[i].shortName != '\0') { fprintf(out, " \\-%c", optable[i].shortName); bol = 0; } if (optable[i].longName != nullptr) { fprintf(out, "%s \\-\\-%s", bol ? "" : ",", optable[i].longName); bol = 0; } } else { if (optable[i].shortName != '\0') { fprintf(out, " \\-%c %s", optable[i].shortName, optable[i].argDescrip); bol = 0; } if (optable[i].longName != nullptr) { fprintf(out, "%s \\-\\-%s=%s", bol ? "" : ",", optable[i].longName, optable[i].argDescrip); bol = 0; } } fprintf(out, "\n%s\n", optable[i].descrip); } } void Subcommand::add_to_optable(std::vector& opts) const { opts.push_back({ "help", '?', 0, 0, 1, "print an help message" }); opts.push_back({ "verbose", 0, POPT_ARG_NONE, (void*)&op_verbose, 0, "verbose output", 0 }); } void Subcommand::manpage_print_options(FILE* out) { string title("Option for command "); title += names[0]; vector opts; add_to_optable(opts); opts.push_back(POPT_TABLEEND); cmdline::manpage_print_options( title.c_str(), opts.data(), out); } poptContext Subcommand::make_popt_context(int argc, const char* argv[], vector& opts) const { add_to_optable(opts); opts.push_back(POPT_TABLEEND); poptContext optCon = poptGetContext(NULL, argc, argv, opts.data(), 0); // Build the help information for this entry string help(usage + "\n\n" + desc + ".\n\n"); if (!longdesc.empty()) help += longdesc + ".\n\n"; help += "Options are:"; poptSetOtherOptionHelp(optCon, help.c_str()); return optCon; } void Subcommand::init() {} Command::~Command() { for (auto& a: ops) delete a; } void Command::add_subcommand(Subcommand* action) { ops.push_back(action); } void Command::add_subcommand(std::unique_ptr&& action) { add_subcommand(action.release()); } Subcommand* Command::find_action(const std::string& name) const { for (auto& a: ops) for (const auto& n: a->names) if (name == n) return a; return nullptr; } void Command::usage(const std::string& selfpath, FILE* out) const { // Get the executable name size_t pos = selfpath.rfind('/'); const char* self = selfpath.c_str(); if (pos != string::npos) self += pos + 1; fprintf(out, "Usage: %s [options] [arguments]\n\n%s.\n%s.\n\n", self, desc.c_str(), longdesc.c_str()); fprintf(out, "Available commands are:\n"); fprintf(out, "\t%s help\n", self); fprintf(out, "\t\tdisplay this help message\n"); fprintf(out, "\t%s help manpage\n", self); fprintf(out, "\t\tgenerate the manpage for %s\n", self); for (auto& a: ops) { bool first = true; for (auto& name: a->names) { if (first) { fprintf(out, "\t%s %s", self, name.c_str()); first = false; } else fprintf(out, " or %s", name.c_str()); } fprintf(out, "\n\t\t%s\n", a->desc.c_str()); } fprintf(out, "\nCalling `%s --help' will give help on the specific command\n", self); } #if 0 void dba_cmdline_print_dba_error() { dba_error_print_to_stderr(); } #endif void error_cmdline::throwf(const char* fmt, ...) { char buf[512]; /* Format the arguments */ va_list ap; va_start(ap, fmt); vsnprintf(buf, 512, fmt, ap); va_end(ap); throw error_cmdline(buf); } void dba_cmdline_error(poptContext optCon, const char* fmt, ...) { va_list ap; va_start(ap, fmt); fputs("Error parsing commandline: ", stderr); vfprintf(stderr, fmt, ap); va_end(ap); fputc('\n', stderr); fputc('\n', stderr); poptPrintHelp(optCon, stderr, 0); exit(1); } Encoding string_to_encoding(const char* type) { if (strlen(type) >= 1) { switch (type[0]) { case 'b': return Encoding::BUFR; case 'c': return Encoding::CREX; } } try { return File::parse_encoding(type); } catch(wreport::error_notfound& e) { error_cmdline::throwf("%s", e.what()); } } void Command::manpage(FILE* out) const { static const char* months[] = { "jan", "feb", "mar", "apr", "may", "jun", "jul", "aug", "sep", "oct", "nov", "dec" }; const char* self = name.c_str(); char* uself; time_t curtime = time(NULL); struct tm* loctime = localtime(&curtime); int i; /* Remove libtool cruft from program name if present */ if (strncmp(self, "lt-", 3) == 0) self += 3; uself = strdup(self); for (i = 0; uself[i] != '\0'; i++) uself[i] = toupper(uself[i]); /* .\" First parameter, NAME, should be all caps .\" Second parameter, SECTION, should be 1-8, maybe w/ subsection .\" other parameters are allowed: see man(7), man(1) .TH APPNAME 1 "may 3, 1976" */ fprintf(out, ".TH %s 1 \"%s %2d, %4d\n", uself, months[loctime->tm_mon], loctime->tm_mday, loctime->tm_year + 1900); /* .\" Some roff macros, for reference: .\" .nh disable hyphenation .\" .hy enable hyphenation .\" .ad l left justify .\" .ad b justify to both left and right margins .\" .nf disable filling .\" .fi enable filling .\" .br insert line break .\" .sp insert n+1 empty lines .\" for manpage-specific macros, see man(7) */ /* .SH NAME debtags \- Manage package tag data in a Debian system */ fprintf(out, ".SH NAME\n" "%s \\- %s\n", self, desc.c_str()); /* .SH SYNOPSIS .B debtags .RI [ options ] .RI [ command ] .RI [ args ... ] .br */ fprintf(out, ".SH SYNOPSIS\n" ".B %s\n" ".RI [ command ]\n" ".RI [ options ]\n" ".RI [ args ... ]\n" ".br\n", self); /* .SH DESCRIPTION \fBdebtags\fP manages package tag data in a debian system and performs basic queries on it. .P Package data are activated or updated in the system using the \fBdebtags [...] */ fprintf(out, ".SH DESCRIPTION\n"); for (i = 0; longdesc[i] != '\0'; i++) switch (longdesc[i]) { case '\n': fprintf(out, "\n.P\n"); break; case '-': fprintf(out, "\\-"); break; default: putc(longdesc[i], out); break; } fprintf(out, ".\n.P\n" "\\fB%s\\fP always requires a non-switch argument, that indicates what is the " "operation that should be performed:\n" ".TP\n" "\\fBhelp\\fP\n" ".br\n" "Print a help summary.\n" ".TP\n" "\\fBhelp manpage\\fP\n" ".br\n" "Print this manpage.\n", self); for (auto& op: ops) { /* .TP \fBupdate\fP .br Download the newest version of the package tag data and update the system package tag database. It needs to be run as root. */ fprintf(out, ".TP\n" "\\fB%s\\fP\n" ".br\n" "%s.\n", op->usage.c_str(), op->desc.c_str()); if (!op->longdesc.empty()) fprintf(out, "%s.\n", op->longdesc.c_str()); } /* .SH OPTIONS \fBdebtags\fP follow the usual GNU command line syntax, with long options starting with two dashes (`-'). */ fprintf(out, ".SH OPTIONS\n" "\\fB%s\\fP follows the usual GNU command line syntax, with long options " "starting with two dashes (`-').\n", self); /* * First trip on all the popt structures to see the subgroups, and document * the subgroups as common stuff. */ { struct poptOption* seen[20]; seen[0] = NULL; for (Subcommand* op: ops) { vector optable; op->add_to_optable(optable); for (auto& sw: optable) if (sw.argInfo == POPT_ARG_INCLUDE_TABLE) { int is_seen = 0; int j; for (j = 0; seen[j] != NULL && j < 20; j++) if (seen[j] == sw.arg) is_seen = 1; if (!is_seen && j < 20) { seen[j] = (struct poptOption*)sw.arg; seen[j+1] = NULL; manpage_print_options(sw.descrip, (struct poptOption*)sw.arg, out); } } } } /* * Then document the rest, without * repeating the options in the subgroups. */ for (auto& op: ops) op->manpage_print_options(out); /* .SH EXAMPLES */ if (!manpage_examples_section.empty()) { fprintf(out, ".SH EXAMPLES\n"); fputs(manpage_examples_section.c_str(), out); } /* .SH FILES */ if (!manpage_files_section.empty()) { fprintf(out, ".SH FILES\n"); fputs(manpage_files_section.c_str(), out); } /* .SH SEE ALSO */ if (!manpage_seealso_section.empty()) { fprintf(out, ".SH SEE ALSO\n"); fputs(manpage_seealso_section.c_str(), out); } fprintf(out, ".SH AUTHOR\n" "\\fB%s\\fP has been written by Enrico Zini " "for ARPA Emilia Romagna, Servizio Idrometeorologico.\n", self); } int Command::main(int argc, const char* argv[]) { int i; /* Dispatch execution to the handler for the various commands */ for (i = 1; i < argc; i++) { /* Check if the user asked for help */ if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "help") == 0) { if (i+1 < argc) { if (strcmp(argv[i+1], "manpage") == 0) manpage(stdout); else { const Subcommand* action = find_action(argv[i+1]); if (action == nullptr) { fputs("Error parsing commandline: ", stderr); fprintf(stderr, "cannot get help on non-existing command '%s'", argv[i+1]); fputc('\n', stderr); fputc('\n', stderr); usage(argv[0], stderr); exit(1); } vector opts; poptContext optCon = action->make_popt_context(argc, argv, opts); poptPrintHelp(optCon, stdout, 0); poptFreeContext(optCon); } } else usage(argv[0], stdout); return 0; } /* Skip switches */ if (argv[i][0] == '-') continue; /* Try the dispatch table */ Subcommand* action = find_action(argv[i]); if (action == nullptr) { usage(argv[0], stderr); return 1; } else { vector opts; poptContext optCon = action->make_popt_context(argc, argv, opts); // Parse commandline int nextOp; while ((nextOp = poptGetNextOpt(optCon)) != -1) { if (nextOp == 1) { poptPrintHelp(optCon, stdout, 0); return 0; } else { dba_cmdline_error(optCon, "invalid flag passed in the commandline"); } } if (action->op_verbose) wreport::notes::set_target(cerr); int res = 0; try { action->init(); res = action->main(optCon); } catch (error_cmdline& e) { fprintf(stderr, "Error parsing commandline: %s\n", e.what()); fputc('\n', stderr); poptPrintHelp(optCon, stderr, 0); res = 1; } catch (std::exception& e) { fprintf(stderr, "%s\n", e.what()); res = 1; } poptFreeContext(optCon); return res; } } /* Nothing found on the dispatch table */ usage(argv[0], stderr); return 1; } unsigned dba_cmdline_get_query(poptContext optCon, Query& query) { core::Query& q = core::Query::downcast(query); unsigned res; const char* queryparm; for (res = 0; (queryparm = poptPeekArg(optCon)) != NULL; ++res) { /* Split the input as name=val */ if (strchr(queryparm, '=') == NULL) break; /* Mark as processed */ poptGetArg(optCon); q.set_from_string(queryparm); } q.validate(); return res; } void list_templates() { const impl::msg::wr::TemplateRegistry& reg = impl::msg::wr::TemplateRegistry::get(); for (impl::msg::wr::TemplateRegistry::const_iterator i = reg.begin(); i != reg.end(); ++i) fprintf(stdout, "%s - %s\n", i->second.name.c_str(), i->second.description.c_str()); } std::list get_filenames(poptContext optCon) { std::list res; while (const char* name = poptGetArg(optCon)) res.push_back(name); return res; } } } dballe-8.6/dballe/cmdline/processor.cc0000644000175000017500000007246013554573614014745 00000000000000#include "processor.h" #include #include #include "dballe/file.h" #include "dballe/message.h" #include "dballe/msg/context.h" #include "dballe/msg/msg.h" #include "dballe/core/csv.h" #include "dballe/core/json.h" #include "dballe/core/match-wreport.h" #include "dballe/cmdline/cmdline.h" #include #include #include #include #include #include #include using namespace wreport; using namespace std; // extern int op_verbose; namespace dballe { namespace cmdline { void ProcessingException::initmsg(const std::string& fname, unsigned index, const char* msg) { char buf[512]; snprintf(buf, 512, "%s:#%u: %s", fname.c_str(), index, msg); this->msg = buf; } static void print_parse_error(const BinaryMessage& msg, error& e) { fprintf(stderr, "Cannot parse %s message #%d: %s at offset %jd.\n", File::encoding_name(msg.encoding), msg.index, e.what(), (intmax_t)msg.offset); } Item::Item() : idx(0), rmsg(0), bulletin(0), msgs(0) { } Item::~Item() { if (msgs) delete msgs; if (bulletin) delete bulletin; if (rmsg) delete rmsg; } void Item::set_msgs(std::vector>* new_msgs) { if (msgs) delete msgs; msgs = new_msgs; } void Item::decode(Importer& imp, bool print_errors) { if (!rmsg) return; if (bulletin) { delete bulletin; bulletin = 0; } if (msgs) { delete msgs; msgs = 0; } // First step: decode raw message to bulletin switch (rmsg->encoding) { case Encoding::BUFR: try { bulletin = BufrBulletin::decode(rmsg->data, rmsg->pathname.c_str(), rmsg->offset).release(); } catch (error& e) { if (print_errors) print_parse_error(*rmsg, e); delete bulletin; bulletin = 0; } break; case Encoding::CREX: try { bulletin = CrexBulletin::decode(rmsg->data, rmsg->pathname.c_str(), rmsg->offset).release(); } catch (error& e) { if (print_errors) print_parse_error(*rmsg, e); delete bulletin; bulletin = 0; } break; } // Second step: decode to msgs switch (rmsg->encoding) { case Encoding::BUFR: case Encoding::CREX: if (bulletin) { msgs = new std::vector>; try { *msgs = imp.from_bulletin(*bulletin); } catch (error& e) { if (print_errors) print_parse_error(*rmsg, e); delete msgs; msgs = 0; } } break; } } void Item::processing_failed(std::exception& e) const { throw ProcessingException(rmsg ? rmsg->pathname : "(unknown)", idx, e); } void IndexMatcher::parse(const std::string& str) { ranges.clear(); str::Split parts(str, ",", true); for (const auto& s: parts) { size_t pos = s.find('-'); if (pos == 0) ranges.push_back(make_pair(0, std::stoi(s.substr(pos + 1)))); else if (pos == s.size() - 1) ranges.push_back(make_pair(std::stoi(s.substr(0, pos)), std::numeric_limits::max())); else if (pos == string::npos) { int val = std::stoi(s); ranges.push_back(make_pair(val, val)); } else ranges.push_back(make_pair(std::stoi(s.substr(0, pos)), std::stoi(s.substr(pos + 1)))); } } bool IndexMatcher::match(int val) const { if (ranges.empty()) return true; for (const auto& range: ranges) if (val >= range.first && val <= range.second) return true; return false; } Filter::Filter() {} Filter::Filter(const ReaderOptions& opts) : category(opts.category), subcategory(opts.subcategory), checkdigit(opts.checkdigit), unparsable(opts.unparsable), parsable(opts.parsable) { if (opts.index_filter) imatcher.parse(opts.index_filter); } Filter::~Filter() { delete matcher; } void Filter::set_index_filter(const std::string& val) { imatcher.parse(val); } void Filter::matcher_reset() { if (matcher) { delete matcher; matcher = 0; } } void Filter::matcher_from_record(const Query& query) { if (matcher) { delete matcher; matcher = 0; } matcher = Matcher::create(query).release(); } bool Filter::match_index(int idx) const { return imatcher.match(idx); } bool Filter::match_common(const BinaryMessage&, const std::vector>* msgs) const { if (msgs == NULL && parsable) return false; if (msgs != NULL && unparsable) return false; return true; } bool Filter::match_bufrex(const BinaryMessage& rmsg, const Bulletin* rm, const std::vector>* msgs) const { if (!match_common(rmsg, msgs)) return false; if (category != -1) if (category != rm->data_category) return false; if (subcategory != -1) if (subcategory != rm->data_subcategory) return false; if (matcher) { if (msgs) { if (!match_msgs(*msgs)) return false; } else if (rm) { if (matcher->match(MatchedBulletin(*rm)) != matcher::MATCH_YES) return false; } } return true; } bool Filter::match_bufr(const BinaryMessage& rmsg, const Bulletin* rm, const std::vector>* msgs) const { if (!match_bufrex(rmsg, rm, msgs)) return false; return true; } bool Filter::match_crex(const BinaryMessage& rmsg, const Bulletin* rm, const std::vector>* msgs) const { if (!match_bufrex(rmsg, rm, msgs)) return false; return true; #if 0 if (grepdata->checkdigit != -1) { int checkdigit; DBA_RUN_OR_RETURN(crex_message_has_check_digit(msg, &checkdigit)); if (grepdata->checkdigit != checkdigit) { *match = 0; return dba_error_ok(); } } #endif } bool Filter::match_msgs(const std::vector>& msgs) const { if (matcher && matcher->match(impl::MatchedMessages(msgs)) != matcher::MATCH_YES) return false; return true; } bool Filter::match_item(const Item& item) const { if (item.rmsg) { switch (item.rmsg->encoding) { case Encoding::BUFR: return match_bufr(*item.rmsg, item.bulletin, item.msgs); case Encoding::CREX: return match_crex(*item.rmsg, item.bulletin, item.msgs); default: return false; } } else if (item.msgs) return match_msgs(*item.msgs); else return false; } Reader::Reader(const ReaderOptions& opts) : input_type(opts.input_type), fail_file_name(opts.fail_file_name), filter(opts) { } bool Reader::has_fail_file() const { return fail_file_name != nullptr; } void Reader::read_csv(const std::list& fnames, Action& action) { // This cannot be implemented in dballe::File at the moment, since // dballe::File reads dballe::BinaryMessage strings, and here we read dballe::Messages // directly. We could split the input into several BinaryMessage strings, but that // would mean parsing the CSV twice: once to detect the message boundaries // and once to parse the BinaryMessage strings. Item item; unique_ptr csvin; list::const_iterator name = fnames.begin(); do { if (name != fnames.end()) { csvin.reset(new CSVReader(*name)); ++name; } else { // name = "(stdin)"; csvin.reset(new CSVReader(cin)); } while (true) { // Read input message unique_ptr msg(new impl::Message); if (!msg->from_csv(*csvin)) break; // Match against index matcher ++item.idx; if (!filter.match_index(item.idx)) continue; // We want it: move it to the item unique_ptr msgs(new impl::Messages); msgs->emplace_back(move(msg)); item.set_msgs(msgs.release()); if (!filter.match_item(item)) continue; action(item); } } while (name != fnames.end()); } void Reader::read_json(const std::list& fnames, Action& action) { using core::JSONParseException; struct JSONMsgReader : public core::JSONReader { std::istream* in; bool close_on_exit; impl::Message msg; std::unique_ptr ctx; std::unique_ptr var; std::unique_ptr attr; enum State { MSG, MSG_IDENT_KEY, MSG_VERSION_KEY, MSG_NETWORK_KEY, MSG_LON_KEY, MSG_LAT_KEY, MSG_DATE_KEY, MSG_DATA_KEY, MSG_DATA_LIST, MSG_DATA_LIST_ITEM, MSG_DATA_LIST_ITEM_VARS_KEY, MSG_DATA_LIST_ITEM_LEVEL_KEY, MSG_DATA_LIST_ITEM_TRANGE_KEY, MSG_DATA_LIST_ITEM_VARS_MAPPING, MSG_DATA_LIST_ITEM_VARS_MAPPING_KEY, MSG_DATA_LIST_ITEM_LEVEL_LIST, MSG_DATA_LIST_ITEM_TRANGE_LIST, MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE1, MSG_DATA_LIST_ITEM_LEVEL_LIST_L1, MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE2, MSG_DATA_LIST_ITEM_LEVEL_LIST_L2, MSG_DATA_LIST_ITEM_TRANGE_LIST_PIND, MSG_DATA_LIST_ITEM_TRANGE_LIST_P1, MSG_DATA_LIST_ITEM_TRANGE_LIST_P2, MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY, MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR, MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_MAPPING, MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_KEY, MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING, MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY, MSG_END, }; std::stack state; JSONMsgReader(std::istream& in) : in(&in), close_on_exit(false) {} JSONMsgReader(const std::string& name) : close_on_exit(true) { in = new ifstream(name); if (not in->good()) throw runtime_error(strerror(errno)); } ~JSONMsgReader() { if (close_on_exit) delete in; } void parse_msgs(std::function cb) { if (in) { while (!in->eof()) { parse(*in); if (not state.empty() && state.top() == MSG_END) { state.pop(); cb(msg); } msg.clear(); } } if (not state.empty()) throw JSONParseException("Incomplete JSON"); } void throw_error_if_empty_state() { if (state.empty()) throw JSONParseException("Invalid JSON value"); } virtual void on_start_list() { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_DATA_KEY: state.pop(); state.push(MSG_DATA_LIST); break; case MSG_DATA_LIST_ITEM_LEVEL_KEY: state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE1); break; case MSG_DATA_LIST_ITEM_TRANGE_KEY: state.pop(); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST_PIND); break; default: throw JSONParseException("Invalid JSON value start_list"); } } virtual void on_end_list() { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_DATA_LIST: state.pop(); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST: state.pop(); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST: state.pop(); break; default: throw JSONParseException("Invalid JSON value end_list"); } } virtual void on_start_mapping() { if (state.empty()) { msg.clear(); state.push(MSG); } else { State s = state.top(); switch (s) { case MSG_DATA_LIST: state.push(MSG_DATA_LIST_ITEM); ctx.reset(new impl::msg::Context(Level(), Trange())); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR: state.pop(); state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_MAPPING); break; case MSG_DATA_LIST_ITEM_VARS_KEY: state.pop(); state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_KEY: state.pop(); state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING); break; default: throw JSONParseException("Invalid JSON value start_mapping"); } } } virtual void on_end_mapping() { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG: { state.pop(); state.push(MSG_END); break; } case MSG_DATA_LIST_ITEM: { // NOTE: station context could be already created, because // of "lon", "lat", "ident", "network". // Then, context overwrite is allowed. // msg.add_context(std::move(ctx)); if (ctx->level.is_missing() && ctx->trange.is_missing()) { msg.station_data.merge(ctx->values); } else { impl::msg::Context& ctx2 = msg.obtain_context(ctx->level, ctx->trange); ctx2.values.merge(ctx->values); } state.pop(); break; } case MSG_DATA_LIST_ITEM_VARS_MAPPING: case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING: case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_MAPPING: state.pop(); break; default: throw JSONParseException("Invalid JSON value end_mapping"); } } virtual void on_add_null() { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_IDENT_KEY: state.pop(); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE1: ctx->level.ltype1 = MISSING_INT; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_L1); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_L1: ctx->level.l1 = MISSING_INT; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE2); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE2: ctx->level.ltype2 = MISSING_INT; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_L2); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_L2: ctx->level.l2 = MISSING_INT; state.pop(); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_PIND: ctx->trange.pind = MISSING_INT; state.pop(); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST_P1); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_P1: ctx->trange.p1 = MISSING_INT; state.pop(); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST_P2); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_P2: ctx->trange.p2 = MISSING_INT; state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY: var->unset(); ctx->values.set(*var); state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY: state.pop(); attr->set(MISSING_INT); var->seta(*attr); ctx->values.set(*var); break; default: throw JSONParseException("Invalid JSON value add_null"); } } virtual void on_add_bool(bool val) { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY: var->set(val); ctx->values.set(*var); state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY: state.pop(); attr->set(val); var->seta(*attr); ctx->values.set(*var); break; default: throw JSONParseException("Invalid JSON value add_bool"); } } virtual void on_add_int(int val) { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_LON_KEY: msg.set_longitude_var(dballe::var("B06001", val)); state.pop(); break; case MSG_LAT_KEY: msg.set_latitude_var(dballe::var("B05001", val)); state.pop(); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE1: ctx->level.ltype1 = val; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_L1); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_L1: ctx->level.l1 = val; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE2); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_LTYPE2: ctx->level.ltype2 = val; state.pop(); state.push(MSG_DATA_LIST_ITEM_LEVEL_LIST_L2); break; case MSG_DATA_LIST_ITEM_LEVEL_LIST_L2: ctx->level.l2 = val; state.pop(); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_PIND: ctx->trange.pind = val; state.pop(); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST_P1); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_P1: ctx->trange.p1 = val; state.pop(); state.push(MSG_DATA_LIST_ITEM_TRANGE_LIST_P2); break; case MSG_DATA_LIST_ITEM_TRANGE_LIST_P2: ctx->trange.p2 = val; state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY: // Var::seti on decimal vars is considered as the value // with the scale already applied var->setf(to_string(val).c_str()); ctx->values.set(*var); state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY: state.pop(); attr->set(val); var->seta(*attr); ctx->values.set(*var); break; default: throw JSONParseException("Invalid JSON value add_int"); } } virtual void on_add_double(double val) { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY: var->set(val); ctx->values.set(*var); state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY: state.pop(); attr->set(val); var->seta(*attr); ctx->values.set(*var); break; default: throw JSONParseException("Invalid JSON value add_double"); } } virtual void on_add_string(const std::string& val) { throw_error_if_empty_state(); State s = state.top(); switch (s) { case MSG: if (val == "ident") state.push(MSG_IDENT_KEY); else if (val == "version") state.push(MSG_VERSION_KEY); else if (val == "network") state.push(MSG_NETWORK_KEY); else if (val == "lon") state.push(MSG_LON_KEY); else if (val == "lat") state.push(MSG_LAT_KEY); else if (val == "date") state.push(MSG_DATE_KEY); else if (val == "data") state.push(MSG_DATA_KEY); else throw JSONParseException("Invalid JSON value"); break; case MSG_IDENT_KEY: msg.set_ident(val.c_str()); state.pop(); break; case MSG_VERSION_KEY: if (strcmp(val.c_str(), DBALLE_JSON_VERSION) != 0) throw JSONParseException("Invalid JSON version " + val); state.pop(); break; case MSG_NETWORK_KEY: msg.set_rep_memo(val.c_str()); state.pop(); break; case MSG_DATE_KEY: msg.set_datetime(Datetime::from_iso8601(val.c_str())); state.pop(); break; case MSG_DATA_LIST_ITEM: if (val == "vars") state.push(MSG_DATA_LIST_ITEM_VARS_KEY); else if (val == "level") state.push(MSG_DATA_LIST_ITEM_LEVEL_KEY); else if (val == "timerange") state.push(MSG_DATA_LIST_ITEM_TRANGE_KEY); else throw JSONParseException("Invalid JSON value"); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING: state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR); var = newvar(val); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_MAPPING: if (val == "v") state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY); else if (val == "a") state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_KEY); else throw JSONParseException("Invalid JSON value"); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_VAR_KEY: var->set(val); ctx->values.set(*var); state.pop(); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING: state.push(MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY); attr = newvar(val); break; case MSG_DATA_LIST_ITEM_VARS_MAPPING_ATTR_MAPPING_VAR_KEY: state.pop(); attr->set(val); var->seta(*attr); ctx->values.set(*var); break; default: throw JSONParseException("Invalid JSON value add_string"); } } }; Item item; std::unique_ptr jsonreader; list::const_iterator name = fnames.begin(); try { do { if (name != fnames.end()) { jsonreader.reset(new JSONMsgReader(*name)); ++name; } else { jsonreader.reset(new JSONMsgReader(cin)); } jsonreader->parse_msgs([&](const impl::Message& msg) { ++item.idx; if (!filter.match_index(item.idx)) return; unique_ptr msgs(new impl::Messages); msgs->emplace_back(make_shared(msg)); item.set_msgs(msgs.release()); if (!filter.match_item(item)) return; action(item); }); } while (name != fnames.end()); } catch (const JSONParseException& e) { // If name points to begin(), then it's the standard input, because // after parsing a file the iterator is incremented. const std::string f = ( name != fnames.begin() ? *(--name) : "stdin" ); throw JSONParseException("Error while parsing JSON from " + f + " at char " + std::to_string(jsonreader->in->tellg()) + ": " + e.what()); } } void Reader::read_file(const std::list& fnames, Action& action) { bool print_errors = !filter.unparsable; std::unique_ptr fail_file; list::const_iterator name = fnames.begin(); do { unique_ptr file; if (input_type == "auto") { if (name != fnames.end()) { file = File::create(*name, "r"); ++name; } else { file = File::create(stdin, false, "standard input"); } } else { Encoding intype = string_to_encoding(input_type.c_str()); if (name != fnames.end()) { file = File::create(intype, *name, "r"); ++name; } else { file = File::create(intype, stdin, false, "standard input"); } } std::unique_ptr imp = Importer::create(file->encoding(), import_opts); while (BinaryMessage bm = file->read()) { Item item; item.rmsg = new BinaryMessage(bm); item.idx = bm.index; bool processed = false; try { // if (op_verbose) // fprintf(stderr, "Reading message #%d...\n", item.index); if (!filter.match_index(item.idx)) continue; try { item.decode(*imp, print_errors); } catch (std::exception& e) { // Convert decode errors into ProcessingException, to skip // this item if it fails to decode. We can safely skip, // because if file->read() returned successfully the next // read should properly start at the next item item.processing_failed(e); } //process_input(*file, rmsg, grepdata, action); if (!filter.match_item(item)) continue; processed = action(item); } catch (ProcessingException& pe) { // If ProcessingException has been raised, we can safely skip // to the next input processed = false; if (verbose) fprintf(stderr, "%s\n", pe.what()); } catch (std::exception& e) { if (verbose) fprintf(stderr, "%s:#%d: %s\n", file->pathname().c_str(), item.idx, e.what()); throw; } // Output items that have not been processed successfully if (!processed && fail_file_name) { if (!fail_file.get()) fail_file = File::create(file->encoding(), fail_file_name, "ab"); fail_file->write(item.rmsg->data); } if (processed) ++count_successes; else ++count_failures; } } while (name != fnames.end()); } void Reader::read(const std::list& fnames, Action& action) { if (input_type == "csv") read_csv(fnames, action); else if (input_type == "json") read_json(fnames, action); else read_file(fnames, action); } } } dballe-8.6/dballe/cmdline/cmdline.h0000644000175000017500000000642513554564112014172 00000000000000#ifndef DBALLE_CMDLINE_CMDLINE_H #define DBALLE_CMDLINE_CMDLINE_H /** @file * @ingroup dballe * Common functions for commandline tools */ #include #include #include #include #include #include #include #include namespace dballe { namespace cmdline { struct Subcommand { std::vector names; std::string usage; std::string desc; std::string longdesc; int op_verbose; virtual ~Subcommand() {} virtual void add_to_optable(std::vector& opts) const; /// Optional initialization before main is called virtual void init(); virtual int main(poptContext) = 0; /** * Create a popt context for this subcommand. * * Options are appended to opts, which is generally passed empty. Its * memory needs to be owned by the caller, because the resulting * poptContext will refer to data inside it, so the lifetime of the vector * should be at least as long as the lifetime of the resulting poptContext. */ poptContext make_popt_context(int argc, const char* argv[], std::vector& opts) const; void manpage_print_options(FILE* out); }; #define ODT_END { NULL, NULL, NULL, NULL, NULL, NULL } struct Command { std::string name; std::string desc; std::string longdesc; std::string manpage_examples_section; std::string manpage_files_section; std::string manpage_seealso_section; std::vector ops; ~Command(); /// Add an action to this tool, taking ownership of its memory management void add_subcommand(Subcommand* action); void add_subcommand(std::unique_ptr&& action); Subcommand* find_action(const std::string& name) const; void usage(const std::string& selfpath, FILE* out) const; void manpage(FILE* out) const; /// Process commandline arguments and perform the action requested int main(int argc, const char* argv[]); }; /// Report an error with command line options struct error_cmdline : public std::exception { std::string msg; ///< error message returned by what() /// @param msg error message error_cmdline(const std::string& msg) : msg(msg) {} ~error_cmdline() throw () {} virtual const char* what() const throw () { return msg.c_str(); } /// Throw the exception, building the message printf-style static void throwf(const char* fmt, ...) WREPORT_THROWF_ATTRS(1, 2); }; /** * Print informations about the last error to stderr */ void dba_cmdline_print_dba_error(); /** * Print an error that happened when parsing commandline arguments, then add * usage informations and exit */ void dba_cmdline_error(poptContext optCon, const char* fmt, ...) __attribute__ ((noreturn)); /** * Return the File::Encoding that corresponds to the name in the string */ Encoding string_to_encoding(const char* type); /** * Get a DB-ALLe query from commandline parameters in the form key=value * * @return the number of key=value pairs found */ unsigned dba_cmdline_get_query(poptContext optCon, Query& query); /** * List available output templates */ void list_templates(); /// Read all the command line arguments and return them as a list std::list get_filenames(poptContext optCon); } } #endif dballe-8.6/dballe/cmdline/dbadb-test.cc0000644000175000017500000000530513554564112014722 00000000000000#include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/cmdline/dbadb.h" #include "dballe/core/arrayfile.h" #include "dballe/msg/msg.h" #include "config.h" using namespace dballe; using namespace dballe::cmdline; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { template class Tests : public FixtureTestCase> { typedef DBFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg2a("cmdline_dbadb_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg6a("cmdline_dbadb_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg8a("cmdline_dbadb_v7_mysql", "MYSQL"); #endif template void Tests::register_tests() { this->add_method("import", [](Fixture& f) { Dbadb dbadb(*f.db); // Import a synop cmdline::ReaderOptions opts; cmdline::Reader reader(opts); wassert(actual(dbadb.do_import(dballe::tests::datafile("bufr/obs0-1.22.bufr"), reader, DBImportOptions::defaults)) == 0); // Export forcing report as temp core::Query query; core::ArrayFile file(Encoding::BUFR); wassert(actual(dbadb.do_export(query, file, "generic", "ship")) == 0); wassert(actual(file.msgs.size()) == 1u); // Decode results auto importer = Importer::create(Encoding::BUFR); impl::Messages msgs = importer->from_binary(file.msgs[0]); wassert(actual(msgs.size()) == 1u); auto msg = impl::Message::downcast(msgs[0]); // Ensure they're ships wassert(actual(msg->type) == MessageType::SHIP); // Check 001194 [SIM] Report mnemonic(CCITTIA5), too const Var* var = msg->get_rep_memo_var(); wassert(actual(var).istrue()); wassert(actual(var->enqc()).istrue()); wassert(actual(var->enq()) == "ship"); }); this->add_method("issue62", [](Fixture& f) { // https://github.com/ARPA-SIMC/dballe/issues/62 Dbadb dbadb(*f.db); // Import a synop cmdline::ReaderOptions opts; cmdline::Reader reader(opts); wassert(actual(dbadb.do_import(dballe::tests::datafile("bufr/issue62.bufr"), reader, DBImportOptions::defaults)) == 0); // Export core::Query query; core::ArrayFile file(Encoding::BUFR); wassert(actual(dbadb.do_export(query, file, "", nullptr)) == 0); wassert(actual(file.msgs.size()) == 2u); // Decode results auto importer = Importer::create(Encoding::BUFR); impl::Messages msgs = importer->from_binary(file.msgs[0]); wassert(actual(msgs.size()) == 1u); auto msg = impl::Message::downcast(msgs[0]); wassert(actual(msg->get_datetime()) == Datetime(2016, 3, 14, 23, 0, 4)); }); } } dballe-8.6/dballe/core/0000755000175000017500000000000013602152021011756 500000000000000dballe-8.6/dballe/core/error.h0000644000175000017500000000117613554564112013223 00000000000000#ifndef DBALLE_CORE_ERROR_H #define DBALLE_CORE_ERROR_H #include namespace dballe { /// Base exception for DB-All.e errors struct error : public wreport::error { /** * String description for an error code. * * It delegates to wreport::error::strerror for codes not known to * DB-All.e, so it can be used instead of wreport::error::strerror. */ static const char* strerror(wreport::ErrorCode code); }; /// Error in case of failed database operations struct error_db : public error { wreport::ErrorCode code() const noexcept override { return wreport::WR_ERR_ODBC; } }; } #endif dballe-8.6/dballe/core/aliases-test.cc0000644000175000017500000000150413554564112014621 00000000000000#include "core/tests.h" #include "core/aliases.h" using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("resolve", []() { wassert(actual_varcode(varcode_alias_resolve("block")) == WR_VAR(0, 1, 1)); wassert(actual_varcode(varcode_alias_resolve("station")) == WR_VAR(0, 1, 2)); wassert(actual_varcode(varcode_alias_resolve("height")) == WR_VAR(0, 7, 30)); wassert(actual_varcode(varcode_alias_resolve("heightbaro")) == WR_VAR(0, 7, 31)); wassert(actual_varcode(varcode_alias_resolve("name")) == WR_VAR(0, 1, 19)); wassert(actual_varcode(varcode_alias_resolve("cippolippo")) == 0); }); } } test("core_aliases"); } dballe-8.6/dballe/core/shortcuts-access.in.cc0000644000175000017500000001131213554564124016126 00000000000000#include "shortcuts.h" #include #include namespace dballe { namespace impl { const Shortcut& Shortcut::by_name(const char* key, unsigned len) { switch (key) { // mklookup case "st_type": return sc::st_type; case "st_name": return sc::st_name; case "st_name_icao": return sc::st_name_icao; case "rep_memo": return sc::rep_memo; case "poll_lcode": return sc::poll_lcode; case "poll_scode": return sc::poll_scode; case "poll_gemscode": return sc::poll_gemscode; case "poll_source": return sc::poll_source; case "poll_atype": return sc::poll_atype; case "poll_ttype": return sc::poll_ttype; case "flight_reg_no": return sc::flight_reg_no; case "flight_phase": return sc::flight_phase; case "flight_roll": return sc::flight_roll; case "navsys": return sc::navsys; case "data_relay": return sc::data_relay; case "wind_inst": return sc::wind_inst; case "temp_precision": return sc::temp_precision; case "latlon_spec": return sc::latlon_spec; case "timesig": return sc::timesig; case "block": return sc::block; case "station": return sc::station; case "ident": return sc::ident; case "year": return sc::year; case "month": return sc::month; case "day": return sc::day; case "hour": return sc::hour; case "minute": return sc::minute; case "second": return sc::second; case "latitude": return sc::latitude; case "longitude": return sc::longitude; case "height_station": return sc::height_station; case "height_baro": return sc::height_baro; case "height_release": return sc::height_release; case "station_height_quality": return sc::station_height_quality; case "isobaric_surface": return sc::isobaric_surface; case "st_dir": return sc::st_dir; case "st_speed": return sc::st_speed; case "meas_equip_type": return sc::meas_equip_type; case "sonde_type": return sc::sonde_type; case "sonde_method": return sc::sonde_method; case "sonde_correction": return sc::sonde_correction; case "sonde_tracking": return sc::sonde_tracking; case "press": return sc::press; case "press_3h": return sc::press_3h; case "press_24h": return sc::press_24h; case "water_temp": return sc::water_temp; case "height_anem": return sc::height_anem; case "press_tend": return sc::press_tend; case "visibility": return sc::visibility; case "pres_wtr": return sc::pres_wtr; case "past_wtr1_3h": return sc::past_wtr1_3h; case "past_wtr1_6h": return sc::past_wtr1_6h; case "past_wtr2_3h": return sc::past_wtr2_3h; case "past_wtr2_6h": return sc::past_wtr2_6h; case "metar_wtr": return sc::metar_wtr; case "tot_prec1": return sc::tot_prec1; case "tot_prec3": return sc::tot_prec3; case "tot_prec6": return sc::tot_prec6; case "tot_prec12": return sc::tot_prec12; case "tot_prec24": return sc::tot_prec24; case "tot_snow": return sc::tot_snow; case "state_ground": return sc::state_ground; case "press_msl": return sc::press_msl; case "qnh": return sc::qnh; case "temp_2m": return sc::temp_2m; case "wet_temp_2m": return sc::wet_temp_2m; case "dewpoint_2m": return sc::dewpoint_2m; case "humidity": return sc::humidity; case "wind_dir": return sc::wind_dir; case "wind_speed": return sc::wind_speed; case "wind_gust_max_speed": return sc::wind_gust_max_speed; case "wind_gust_max_dir": return sc::wind_gust_max_dir; case "ex_ccw_wind": return sc::ex_ccw_wind; case "ex_cw_wind": return sc::ex_cw_wind; case "cloud_n": return sc::cloud_n; case "cloud_nh": return sc::cloud_nh; case "cloud_hh": return sc::cloud_hh; case "cloud_cl": return sc::cloud_cl; case "cloud_cm": return sc::cloud_cm; case "cloud_ch": return sc::cloud_ch; case "cloud_n1": return sc::cloud_n1; case "cloud_c1": return sc::cloud_c1; case "cloud_h1": return sc::cloud_h1; case "cloud_n2": return sc::cloud_n2; case "cloud_c2": return sc::cloud_c2; case "cloud_h2": return sc::cloud_h2; case "cloud_n3": return sc::cloud_n3; case "cloud_c3": return sc::cloud_c3; case "cloud_h3": return sc::cloud_h3; case "cloud_n4": return sc::cloud_n4; case "cloud_c4": return sc::cloud_c4; case "cloud_h4": return sc::cloud_h4; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } } } dballe-8.6/dballe/core/values.cc0000644000175000017500000000573213554564112013531 00000000000000#include "values.h" #include "dballe/core/var.h" #include #include using namespace std; using namespace wreport; namespace dballe { namespace core { namespace value { Encoder::Encoder() { buf.reserve(64); } void Encoder::append_uint16(uint16_t val) { uint16_t encoded = htons(val); buf.insert(buf.end(), (uint8_t*)&encoded, (uint8_t*)&encoded + 2); } void Encoder::append_uint32(uint32_t val) { uint32_t encoded = htonl(val); buf.insert(buf.end(), (uint8_t*)&encoded, (uint8_t*)&encoded + 4); } void Encoder::append_cstring(const char* val) { for ( ; *val; ++val) buf.push_back(*val); buf.push_back(0); } void Encoder::append(const wreport::Var& var) { // Encode code append_uint16(var.code()); switch (var.info()->type) { case Vartype::Binary: case Vartype::String: // Encode value, including terminating zero append_cstring(var.enqc()); break; case Vartype::Integer: case Vartype::Decimal: // Just encode the integer value append_uint32(var.enqi()); break; } } void Encoder::append_attributes(const wreport::Var& var) { for (const Var* a = var.next_attr(); a != NULL; a = a->next_attr()) append(*a); } Decoder::Decoder(const std::vector& buf) : buf(buf.data()), size(buf.size()) {} uint16_t Decoder::decode_uint16() { if (size < 2) error_toolong::throwf("cannot decode a 16 bit integer: only %u bytes are left to read", size); uint16_t res = ntohs(*(uint16_t*)buf); buf += 2; size -= 2; return res; } uint32_t Decoder::decode_uint32() { if (size < 4) error_toolong::throwf("cannot decode a 32 bit integer: only %u bytes are left to read", size); uint32_t res = ntohl(*(uint32_t*)buf); buf += 4; size -= 4; return res; } const char* Decoder::decode_cstring() { if (!size) error_toolong::throwf("cannot decode a C string: the buffer is empty"); const char* res = (const char*)buf; while (true) { if (!size) error_toolong::throwf("cannot decode a C string: reached the end of buffer before finding the string terminator"); if (*buf == 0) break; ++buf; --size; } ++buf; --size; return res; } unique_ptr Decoder::decode_var() { wreport::Varinfo info = varinfo(decode_uint16()); switch (info->type) { case Vartype::Binary: case Vartype::String: return unique_ptr(new wreport::Var(info, decode_cstring())); case Vartype::Integer: case Vartype::Decimal: return unique_ptr(new wreport::Var(info, (int)decode_uint32())); default: error_consistency::throwf("unsupported variable type %d", (int)info->type); } } void Decoder::decode_attrs(const std::vector& buf, wreport::Var& var) { Decoder dec(buf); while (dec.size) var.seta(move(dec.decode_var())); } } } } dballe-8.6/dballe/core/file.cc0000644000175000017500000000450013554564112013141 00000000000000#include "file.h" #include #include #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace core { File::File(const std::string& name, FILE* fd, bool close_on_exit) : m_name(name), fd(fd), close_on_exit(close_on_exit), idx(0) { } File::~File() { close(); } void File::close() { if (fd && close_on_exit) { fclose(fd); fd = nullptr; } } bool File::foreach(std::function dest) { while (true) { if (auto bm = read()) { if (!dest(bm)) return false; } else break; } return true; } std::string File::resolve_test_data_file(const std::string& name) { // Skip appending the test data path for pathnames starting with ./ if (name[0] == '.') return name; const char* testdatadirenv = getenv("DBA_TESTDATA"); std::string testdatadir = testdatadirenv ? testdatadirenv : "."; return testdatadir + "/" + name; } std::unique_ptr File::open_test_data_file(Encoding type, const std::string& name) { return File::create(type, resolve_test_data_file(name), "r"); } BinaryMessage BufrFile::read() { if (fd == nullptr) throw error_consistency("cannot read from a closed file"); BinaryMessage res(Encoding::BUFR); if (BufrBulletin::read(fd, res.data, m_name.c_str(), &res.offset)) { res.pathname = m_name; res.index = idx++; return res; } return BinaryMessage(Encoding::BUFR); } void BufrFile::write(const std::string& msg) { if (fd == nullptr) throw error_consistency("cannot write to a closed file"); BufrBulletin::write(msg, fd, m_name.c_str()); } BinaryMessage CrexFile::read() { if (fd == nullptr) throw error_consistency("cannot read from a closed file"); BinaryMessage res(Encoding::CREX); if (CrexBulletin::read(fd, res.data, m_name.c_str(), &res.offset)) { res.pathname = m_name; res.index = idx++; return res; } return BinaryMessage(Encoding::CREX); } void CrexFile::write(const std::string& msg) { if (fd == nullptr) throw error_consistency("cannot write to a closed file"); CrexBulletin::write(msg, fd, m_name.c_str()); } } } dballe-8.6/dballe/core/shortcuts-test.cc0000644000175000017500000000161113554564112015235 00000000000000#include "tests.h" #include "shortcuts.h" using namespace dballe; using namespace dballe::impl; using namespace dballe::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { // Test variable alias resolution add_method("shortcuts", []() { // First wassert(actual(Shortcut::by_name("block")) == sc::block); wassert(actual(Shortcut::by_name("blocks", 5)) == sc::block); // Last wassert(actual(Shortcut::by_name("tot_prec1")) == sc::tot_prec1); // Inbetween wassert(actual(Shortcut::by_name("cloud_h4")) == sc::cloud_h4); wassert(actual(Shortcut::by_name("st_type")) == sc::st_type); wassert(actual(Shortcut::by_name("tot_snow")) == sc::tot_snow); }); } } test("msg_vars"); } dballe-8.6/dballe/core/error.cc0000644000175000017500000000021713554564112013354 00000000000000#include "error.h" namespace dballe { const char* error::strerror(wreport::ErrorCode code) { return wreport::error::strerror(code); } } dballe-8.6/dballe/core/query.cc0000644000175000017500000004120313572423710013366 00000000000000#include "query.h" #include "var.h" #include "json.h" #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace core { void Query::validate() { lonrange.set(lonrange); dtrange.min.set_lower_bound(); dtrange.max.set_upper_bound(); } std::unique_ptr Query::clone() const { return unique_ptr(new Query(*this)); } const Query& Query::downcast(const dballe::Query& query) { const Query* ptr = dynamic_cast(&query); if (!ptr) throw error_consistency("query given is not a core::Query"); return *ptr; } Query& Query::downcast(dballe::Query& query) { Query* ptr = dynamic_cast(&query); if (!ptr) throw error_consistency("query given is not a core::Query"); return *ptr; } unsigned Query::get_modifiers() const { return parse_modifiers(query.c_str()); } void Query::clear() { ana_id = MISSING_INT; priomin = MISSING_INT; priomax = MISSING_INT; report.clear(); mobile = MISSING_INT; ident.clear(); latrange = LatRange(); lonrange = LonRange(); dtrange = DatetimeRange(); level = Level(); trange = Trange(); varcodes.clear(); query.clear(); ana_filter.clear(); data_filter.clear(); attr_filter.clear(); limit = MISSING_INT; block = MISSING_INT; station = MISSING_INT; } void Query::set_from_string(const char* str) { // Split the input as name=val const char* s = strchr(str, '='); if (!s) error_consistency::throwf("there should be an = between the name and the value in '%s'", str); string key(str, s - str); const char* val = s + 1; if (strcmp(val, "-") == 0) unset(key.data(), key.size()); else setf(key.data(), key.size(), val); } void Query::set_from_test_string(const std::string& s) { clear(); if (s.empty()) return; size_t cur = 0; while (true) { size_t next = s.find(", ", cur); if (next == string::npos) { set_from_string(s.substr(cur).c_str()); break; } else { set_from_string(s.substr(cur, next - cur).c_str()); cur = next + 2; } } validate(); } namespace { bool removed_or_changed(int val, int other) { // if (val == other) return false; // Not changed // if (val != MISSING_INT && other == MISSING_INT) return false; // Added filter // if (val == MISSING_INT && other != MISSING_INT) return true; // Removed filter // if (val != other) return true; // Changed return !(other == val || other == MISSING_INT); } bool removed_or_changed(const Ident& val, const Ident& other) { return !(other == val || other.is_missing()); } bool removed_or_changed(const std::string& val, const std::string& other) { return !(other == val || other.empty()); } // Return true if sub is a subset of sup, or the same as sup template bool is_subset(const std::set& sub, const std::set& sup) { auto isub = sub.begin(); auto isup = sup.begin(); while (isub != sub.end() && isup != sup.end()) { // A common item: ok. Skip it and continue. if (*isub == *isup) { ++isub; ++isup; continue; } // sub contains an item not in sup: not a subset. if (*isub < *isup) return false; // sup contains an item not in sub: ok. Skip it and continue. ++isup; } // If we have seen all of sub so far, then it is a subset. return isub == sub.end(); } // Return true if sub1--sub2 is contained in sup1--sup2, or is the same bool is_subrange(int sub1, int sub2, int sup1, int sup2) { // If sup is the whole domain, sub is contained in it if (sup1 == MISSING_INT && sup2 == MISSING_INT) return true; // If sup is left-open, only check the right bounds if (sup1 == MISSING_INT) return sub2 != MISSING_INT && sub2 <= sup2; // If sup is right-open, only check the left bounds if (sup2 == MISSING_INT) return sub1 != MISSING_INT && sub1 >= sup1; // sup is bounded both ways if (sub1 == MISSING_INT || sub1 < sup1) return false; if (sub2 == MISSING_INT || sub2 > sup2) return false; return true; } } bool Query::is_subquery(const dballe::Query& other_gen) const { const Query& other = downcast(other_gen); if (removed_or_changed(ana_id, other.ana_id)) return false; if (!is_subrange(priomin, priomax, other.priomin, other.priomax)) return false; if (removed_or_changed(report, other.report)) return false; if (removed_or_changed(mobile, other.mobile)) return false; if (removed_or_changed(ident, other.ident)) return false; if (!other.latrange.contains(latrange)) return false; if (!other.lonrange.contains(lonrange)) return false; if (!other.dtrange.contains(dtrange)) return false; if (removed_or_changed(level.ltype1, other.level.ltype1)) return false; if (removed_or_changed(level.l1, other.level.l1)) return false; if (removed_or_changed(level.ltype2, other.level.ltype2)) return false; if (removed_or_changed(level.l2, other.level.l2)) return false; if (removed_or_changed(trange.pind, other.trange.pind)) return false; if (removed_or_changed(trange.p1, other.trange.p1)) return false; if (removed_or_changed(trange.p2, other.trange.p2)) return false; // If other.varcodes is a subset, of varcodes, then we just added new // varcodes to the filter if (!is_subset(other.varcodes, varcodes)) return false; // Parse query and check its components unsigned mods = parse_modifiers(query.c_str()); unsigned omods = parse_modifiers(other.query.c_str()); if (mods != omods) { // The only relevant bits is query=best, all the rest we can safely ignore if (!(mods & DBA_DB_MODIFIER_BEST) && (omods & DBA_DB_MODIFIER_BEST)) return false; } if (removed_or_changed(ana_filter, other.ana_filter)) return false; if (removed_or_changed(data_filter, other.data_filter)) return false; if (removed_or_changed(attr_filter, other.attr_filter)) return false; // We tolerate limit being changed as long as it has only been reduced if (other.limit != MISSING_INT && (limit == MISSING_INT || limit > other.limit)) return false; if (removed_or_changed(block, other.block)) return false; if (removed_or_changed(station, other.station)) return false; return true; } namespace { struct Printer { const Query& q; FILE* out; bool first = true; Printer(const Query& q, FILE* out) : q(q), out(out) {} void print_int(const char* name, int val) { if (val == MISSING_INT) return; if (!first) fputs(", ", out); fprintf(out, "%s=%d", name, val); first = false; } void print_str(const char* name, bool is_present, const std::string& val) { if (!is_present) return; if (!first) fputs(", ", out); fprintf(out, "%s=%s", name, val.c_str()); first = false; } void print_ident(const Ident& val) { if (val.is_missing()) return; if (!first) fputs(", ", out); fprintf(out, "ident=%s", val.get()); first = false; } void print_latrange(const LatRange& latrange) { if (latrange.is_missing()) return; double dmin, dmax; latrange.get(dmin, dmax); if (dmin != LatRange::DMIN) { if (!first) fputs(", ", out); fprintf(out, "latmin=%.5f", dmin); first = false; } if (dmax != LatRange::DMAX) { if (!first) fputs(", ", out); fprintf(out, "latmax=%.5f", dmax); first = false; } } void print_lonrange(const LonRange& lonrange) { if (lonrange.is_missing()) return; double dmin, dmax; lonrange.get(dmin, dmax); if (!first) fputs(", ", out); fprintf(out, "lonmin=%.5f, lonmax=%.5f", dmin, dmax); first = false; } void print_datetimerange(const DatetimeRange& dtr) { if (dtr.is_missing()) return; if (!first) fputs(", ", out); if (dtr.min == dtr.max) fprintf(out, "datetime=%04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu", dtr.max.year, dtr.max.month, dtr.max.day, dtr.max.hour, dtr.max.minute, dtr.max.second); else if (dtr.min.is_missing()) fprintf(out, "datetime<=%04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu", dtr.max.year, dtr.max.month, dtr.max.day, dtr.max.hour, dtr.max.minute, dtr.max.second); else if (dtr.max.is_missing()) fprintf(out, "datetime>=%04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu", dtr.min.year, dtr.min.month, dtr.min.day, dtr.min.hour, dtr.min.minute, dtr.min.second); else fprintf(out, "datetime=%04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu to %04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu", dtr.min.year, dtr.min.month, dtr.min.day, dtr.min.hour, dtr.min.minute, dtr.min.second, dtr.max.year, dtr.max.month, dtr.max.day, dtr.max.hour, dtr.max.minute, dtr.max.second); first = false; } void print_level(const char* name, const Level& l) { if (l == Level()) return; if (!first) fputs(", ", out); fprintf(out, "%s=", name); l.print(out, "-", ""); first = false; } void print_trange(const char* name, const Trange& t) { if (t == Trange()) return; if (!first) fputs(", ", out); fprintf(out, "%s=", name); t.print(out, "-", ""); first = false; } void print_varcodes(const char* name, const set& varcodes) { if (varcodes.empty()) return; if (!first) fputs(", ", out); fputs(name, out); putc('=', out); bool lfirst = true; for (const auto& v : varcodes) { if (!lfirst) fputs(",", out); fprintf(out, "%01d%02d%03d", WR_VAR_F(v), WR_VAR_X(v), WR_VAR_Y(v)); lfirst = false; } first = false; } void print() { print_int("ana_id", q.ana_id); print_int("priomin", q.priomin); print_int("priomax", q.priomax); print_str("report", !q.report.empty(), q.report); print_int("mobile", q.mobile); print_ident(q.ident); print_latrange(q.latrange); print_lonrange(q.lonrange); print_datetimerange(q.dtrange); print_level("level", q.level); print_trange("trange", q.trange); print_varcodes("varcodes", q.varcodes); print_str("query", !q.query.empty(), q.query); print_str("ana_filter", !q.ana_filter.empty(), q.ana_filter); print_str("data_filter", !q.data_filter.empty(), q.data_filter); print_str("attr_filter", !q.attr_filter.empty(), q.attr_filter); print_int("limit", q.limit); print_int("block", q.block); print_int("station", q.station); putc('\n', out); } }; } void Query::print(FILE* out) const { Printer printer(*this, out); printer.print(); } void Query::serialize(JSONWriter& out) const { if (ana_id != MISSING_INT) out.add("ana_id", ana_id); if (priomin != MISSING_INT) out.add("prio_min", priomin); if (priomax != MISSING_INT) out.add("prio_max", priomax); if (!report.empty()) out.add("rep_memo", report); if (mobile != MISSING_INT) out.add("mobile", mobile); if (!ident.is_missing()) out.add("ident", ident.get()); if (!latrange.is_missing()) { if (latrange.imin != LatRange::IMIN) out.add("latmin", latrange.imin); if (latrange.imax != LatRange::IMAX) out.add("latmax", latrange.imax); } if (!lonrange.is_missing()) { out.add("lonmin", lonrange.imin); out.add("lonmax", lonrange.imax); } if (dtrange.min == dtrange.max) { if (!dtrange.min.is_missing()) out.add("datetime", dtrange.min); } else { if (!dtrange.min.is_missing()) out.add("datetime_min", dtrange.min); if (!dtrange.max.is_missing()) out.add("datetime_max", dtrange.max); } if (!level.is_missing()) out.add("level", level); if (!trange.is_missing()) out.add("trange", trange); if (!varcodes.empty()) { out.add("varcodes"); out.add_list(varcodes); } if (!query.empty()) out.add("query", query); if (!ana_filter.empty()) out.add("ana_filter", ana_filter); if (!data_filter.empty()) out.add("data_filter", data_filter); if (!attr_filter.empty()) out.add("attr_filter", attr_filter); if (limit != MISSING_INT) out.add("limit", limit); if (block != MISSING_INT) out.add("block", block); if (station != MISSING_INT) out.add("station", station); } unsigned Query::parse_modifiers(const char* s) { unsigned modifiers = 0; while (*s) { size_t len = strcspn(s, ","); int got = 1; switch (len) { case 0: /* If it's an empty token, skip it */ break; case 4: /* "best": if more values exist in a point, get only the best one */ if (strncmp(s, "best", 4) == 0) modifiers |= DBA_DB_MODIFIER_BEST; else got = 0; break; case 5: if (strncmp(s, "attrs", 4) == 0) modifiers |= DBA_DB_MODIFIER_WITH_ATTRIBUTES; else got = 0; break; case 6: /* "bigana": optimize with date first */ if (strncmp(s, "bigana", 6) == 0) ; // Not used anymore else if (strncmp(s, "nosort", 6) == 0) modifiers |= DBA_DB_MODIFIER_UNSORTED; else if (strncmp(s, "stream", 6) == 0) ; // Not used anymore else got = 0; break; case 7: if (strncmp(s, "details", 7) == 0) modifiers |= DBA_DB_MODIFIER_SUMMARY_DETAILS; else got = 0; break; default: got = 0; break; } /* Check that we parsed it correctly */ if (!got) error_consistency::throwf("Query modifier \"%.*s\" is not recognized", (int)len, s); /* Move to the next token */ s += len; if (*s == ',') ++s; } return modifiers; } Query Query::from_json(core::json::Stream& in) { Query res; in.parse_object([&](const std::string& key) { if (key == "ana_id") res.ana_id = in.parse_signed(); else if (key == "prio_min") res.priomin = in.parse_signed(); else if (key == "prio_max") res.priomax = in.parse_signed(); else if (key == "rep_memo") res.report = in.parse_string(); else if (key == "mobile") res.mobile = in.parse_signed(); else if (key == "ident") res.ident = in.parse_string(); else if (key == "latmin") res.latrange.imin = in.parse_signed(); else if (key == "latmax") res.latrange.imax = in.parse_signed(); else if (key == "lonmin") res.lonrange.imin = in.parse_signed(); else if (key == "lonmax") res.lonrange.imax = in.parse_signed(); else if (key == "datetime") res.dtrange.max = res.dtrange.min = in.parse_datetime(); else if (key == "datetime_min") res.dtrange.min = in.parse_datetime(); else if (key == "datetime_max") res.dtrange.max = in.parse_datetime(); else if (key == "level") res.level = in.parse_level(); else if (key == "trange") res.trange = in.parse_trange(); else if (key == "varcodes") { in.parse_array([&]{ res.varcodes.insert(in.parse_unsigned()); }); } else if (key == "query") res.query = in.parse_string(); else if (key == "ana_filter") res.ana_filter = in.parse_string(); else if (key == "data_filter") res.data_filter = in.parse_string(); else if (key == "attr_filter") res.attr_filter = in.parse_string(); else if (key == "limit") res.limit = in.parse_signed(); else if (key == "block") res.block = in.parse_signed(); else if (key == "station") res.station = in.parse_signed(); }); return res; } } } dballe-8.6/dballe/core/matcher.h0000644000175000017500000000463113554564112013514 00000000000000#ifndef DBALLE_CORE_MATCHER_H #define DBALLE_CORE_MATCHER_H #include #include #include namespace dballe { namespace matcher { enum Result { MATCH_YES, // Item matches MATCH_NO, // Item does not match MATCH_NA // Match not applicable to this item }; /// Format a Result into a string std::string result_format(Result res); } /** * Common interface for things that are matched. * * This allows the Record-derived matcher to operate on several different * elements. Examples are Record and Msg, but can also be unknown elements * provided by code that uses DB-All.e. */ struct Matched { virtual ~Matched() {} /** * Match variable ID * * This corresponds to B33195 */ virtual matcher::Result match_var_id(int val) const; /** * Match station ID * * This corresponds to DBA_KEY_ANA_ID */ virtual matcher::Result match_station_id(int val) const; /** * Match station WMO code * * If station is -1, only match the block. */ virtual matcher::Result match_station_wmo(int block, int station=-1) const; /// Match datetime virtual matcher::Result match_datetime(const DatetimeRange& range) const; /** * Match coordinates, with bounds in 1/100000 of degree * * Any value can be set to MISSING_INT if not applicable or to represent an * open bound */ virtual matcher::Result match_coords(const LatRange& latrange, const LonRange& lonrange) const; /** * Match rep_memo * * the memo value that is passed is always lowercase */ virtual matcher::Result match_rep_memo(const char* memo) const; /** * Match if min <= val <= max * * It correctly deals with min and max being set to MISSING_INT to signify an open * bound. */ static matcher::Result int_in_range(int val, int min, int max); /** * Match if val is contained inside the given longitude range */ static matcher::Result lon_in_range(int val, int min, int max); }; /** * Match DB-All.e objects using the same queries that can be made on DB-All.e * databases. */ struct Matcher { virtual ~Matcher() {} virtual matcher::Result match(const Matched& item) const = 0; virtual void to_query(dballe::core::Query& query) const = 0; static std::unique_ptr create(const dballe::Query& query); }; } #endif dballe-8.6/dballe/core/cursor.cc0000644000175000017500000000515013572414716013545 00000000000000#include "cursor.h" #include "dballe/types.h" #include namespace dballe { namespace impl { namespace { template struct EmptyCursor : public Interface { bool has_value() const { return false; } void enq(Enq& enq) const {} int remaining() const override { return 0; } bool next() override { return false; } void discard() override {} DBStation get_station() const override { return DBStation(); } }; struct EmptyCursorStation : public EmptyCursor { DBValues get_values() const override { return DBValues(); } }; struct EmptyCursorStationData : public EmptyCursor { wreport::Varcode get_varcode() const override { return 0; } wreport::Var get_var() const { throw wreport::error_consistency("cursor not on a result"); } }; struct EmptyCursorData : public EmptyCursor { wreport::Varcode get_varcode() const override { return 0; } wreport::Var get_var() const { throw wreport::error_consistency("cursor not on a result"); } Level get_level() const override { return Level(); } Trange get_trange() const override { return Trange(); } Datetime get_datetime() const override { return Datetime(); } }; struct EmptyCursorSummary : public EmptyCursor { Level get_level() const override { return Level(); } Trange get_trange() const override { return Trange(); } wreport::Varcode get_varcode() const override { return 0; } DatetimeRange get_datetimerange() const override { return DatetimeRange(); } size_t get_count() const override { return 0; } }; struct EmptyCursorMessage : public EmptyCursor { const Message& get_message() const override { throw wreport::error_notfound("cannot retrieve a message from an empty result set"); } std::unique_ptr detach_message() override { throw wreport::error_notfound("cannot retrieve a message from an empty result set"); } }; } std::unique_ptr CursorStation::make_empty() { return std::unique_ptr(new EmptyCursorStation); } std::unique_ptr CursorStationData::make_empty() { return std::unique_ptr(new EmptyCursorStationData); } std::unique_ptr CursorData::make_empty() { return std::unique_ptr(new EmptyCursorData); } std::unique_ptr CursorSummary::make_empty() { return std::unique_ptr(new EmptyCursorSummary); } std::unique_ptr CursorMessage::make_empty() { return std::unique_ptr(new EmptyCursorMessage); } } } dballe-8.6/dballe/core/benchmark.h0000644000175000017500000000720613554564112014024 00000000000000#ifndef DBALLE_CORE_BENCHMARK_H #define DBALLE_CORE_BENCHMARK_H /** @file * Simple benchmark infrastructure. */ #include #include #include #include #include #include #include #include #include #include namespace dballe { namespace benchmark { struct Benchmark; /// One task to be measured. struct Task { Task() {} Task(const Task&) = delete; Task(Task&&) = delete; virtual ~Task() {} Task& operator=(const Task&) = delete; Task& operator=(Task&&) = delete; virtual const char* name() const = 0; /// Set up the environment for running run_once() virtual void setup() {} /** * Run the task once. * * It can be called multiple times bewteen setup and teardown in order to * perform repeated measurements. */ virtual void run_once() = 0; /// Clean up after the task has been measured. virtual void teardown() {} }; struct Progress; struct Timeit { std::string task_name; /// How many times to repeat the task for measuring how long it takes unsigned repetitions = 1; struct timespec time_at_start; struct timespec time_at_end; struct rusage res_at_start; struct rusage res_at_end; void run(Progress& progress, Task& task); }; struct Throughput { std::string task_name; /// How many seconds to run the task to see how many times per second it runs double run_time = 0.5; unsigned times_run = 0; void run(Progress& progress, Task& task); }; /// Notify of progress during benchmark execution struct Progress { virtual ~Progress() {} virtual void start_timeit(const Timeit& t) = 0; virtual void end_timeit(const Timeit& t) = 0; virtual void start_throughput(const Throughput& t) = 0; virtual void end_throughput(const Throughput& t) = 0; virtual void test_failed(const Task& t, std::exception& e) = 0; }; /** * Basic progress implementation writing progress information to the given * output stream */ struct BasicProgress : Progress { FILE* out; FILE* err; BasicProgress(FILE* out=stdout, FILE* err=stderr); void start_timeit(const Timeit& t) override; void end_timeit(const Timeit& t) override; void start_throughput(const Throughput& t) override; void end_throughput(const Throughput& t) override; void test_failed(const Task& t, std::exception& e) override; }; /** * Base class for all benchmarks. */ struct Benchmark { /// Progress indicator std::shared_ptr progress; /// Tasks for which we time their duration std::vector timeit_tasks; /// Tasks for which we time their throughput std::vector throughput_tasks; Benchmark(); virtual ~Benchmark(); /// Run the benchmark and collect timings void timeit(Task& task, unsigned repetitions=1); /// Run the benchmark and collect timings void throughput(Task& task, double run_time=0.5); /// Print timings to stdout void print_timings(); }; /** * Container for parsed messages used for benchmarking */ struct Messages : public std::vector>> { void load(const std::string& pathname, dballe::Encoding encoding=dballe::Encoding::BUFR, const char* codec_options="accurate"); // Copy the first \a size messages, change their datetime, and append them // to the vector void duplicate(size_t size, const Datetime& datetime); }; struct Whitelist : protected std::vector { Whitelist(int argc, const char* argv[]); bool has(const std::string& val); }; } } #endif dballe-8.6/dballe/core/structbuf.h0000644000175000017500000000737313554564112014120 00000000000000/* * core/structbuf - memory or file-backed storage of structures * * Copyright (C) 2014 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ #ifndef DBALLE_CORE_STRUCTBUF_H #define DBALLE_CORE_STRUCTBUF_H #include #include #include #include namespace dballe { namespace structbuf { int make_anonymous_tmpfile(); void write_buffer(int fd, void* buf, size_t size); } /** * Buffer of simple structures that becomes file backed if it grows beyond a * certain size. * * bufsize is the number of T items that we keep in memory before becoming * file-backed. */ template class Structbuf { protected: /** * In-memory buffer using during appending. When it becomes full, it is * flushed out to a temporary file. */ T* membuf = nullptr; /// Number of items in membuf unsigned membuf_last = 0; /** * Memory area used for reading. It points to membuf if we are * memory-backed, or it is the mmap view of the file if we are file-backed */ const T* readbuf = (const T*)MAP_FAILED; /// Number of items appended so far size_t m_count = 0; /// Unix file descriptor to the temporary file, or -1 if we are memory /// backed int tmpfile_fd = -1; public: Structbuf() : membuf(new T[bufsize]) { } ~Structbuf() { delete[] membuf; if (tmpfile_fd != -1) { if (readbuf != MAP_FAILED) munmap((void*)readbuf, m_count * sizeof(T)); ::close(tmpfile_fd); } } /// Get the number of structures that have been added to the buffer so far size_t size() const { return m_count; } /// Return true if the buffer has become file-backed bool is_file_backed() const { return tmpfile_fd != -1; } /// Append an item to the buffer void append(const T& val) { if (readbuf != MAP_FAILED) throw wreport::error_consistency("writing to a Structbuf that is already being read"); if (membuf_last == bufsize) write_to_file(); membuf[membuf_last++] = val; ++m_count; } /// Stop appending and get ready to read back the data void ready_to_read() { if (tmpfile_fd == -1) readbuf = membuf; else { // Flush the remaining memory data to file if (membuf_last) write_to_file(); // mmap the file for reading readbuf = (const T*)mmap(nullptr, m_count * sizeof(T), PROT_READ, MAP_SHARED, tmpfile_fd, 0); if (readbuf == MAP_FAILED) throw wreport::error_system("cannot map temporary file contents to memory"); } } /// Read back an item const T& operator[](size_t idx) const { return readbuf[idx]; } protected: void write_to_file() { if (tmpfile_fd == -1) tmpfile_fd = structbuf::make_anonymous_tmpfile(); structbuf::write_buffer(tmpfile_fd, membuf, sizeof(T) * membuf_last); membuf_last = 0; } }; } #endif dballe-8.6/dballe/core/structbuf.cc0000644000175000017500000000355613554564112014255 00000000000000/* * core/structbuf - memory or file-backed storage of structures * * Copyright (C) 2014 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ #include "structbuf.h" #include #include using namespace std; using namespace wreport; namespace dballe { namespace structbuf { int make_anonymous_tmpfile() { const char* tmpdir = getenv("TMPDIR"); if (!tmpdir) tmpdir = "/tmp"; std::string tmpnam = tmpdir; tmpnam += "/dballe-XXXXXX"; // FIXME: to avoid a string copy, we are writing directly to the string buffer int fd = mkstemp((char*)tmpnam.c_str()); if (fd == -1) error_system::throwf("cannot create temporary file in %s", tmpdir); if (unlink(tmpnam.c_str()) == -1) { int orig_errno = errno; close(fd); throw error_system("cannot remove newly created temporary file", orig_errno); } return fd; } void write_buffer(int fd, void* buf, size_t size) { ssize_t res = ::write(fd, buf, size); if (res == -1) throw error_system("cannot write to temporary file"); if ((size_t)res < size) throw error_consistency("write to temporary file was interrupted"); } } } dballe-8.6/dballe/core/matcher-test.cc0000644000175000017500000002055413554573614014640 00000000000000#include "tests.h" #include "matcher.h" #include #include using namespace std; using namespace dballe; using namespace dballe::core; using namespace dballe::tests; namespace { #if 0 std::unique_ptr get_matcher(const char* q) { return Matcher::create(*dballe::tests::query_from_string(q)); } #endif class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_matcher"); void Tests::register_tests() { add_method("empty", []() { }); #if 0 // Test station_id matcher add_method("station_id", []() { auto m = get_matcher("ana_id=1"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.id = 2; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.id = 1; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); }); // Test station WMO matcher add_method("station_wmo", []() { { auto m = get_matcher("block=11"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 1)).set(1); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 1)).set(11); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.obtain(WR_VAR(0, 1, 2)).set(222); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("block=11, station=222"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 1)).set(1); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 1)).set(11); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 2)).set(22); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.obtain(WR_VAR(0, 1, 2)).set(222); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.obtain(WR_VAR(0, 1, 1)).set(1); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.unset_var(WR_VAR(0, 1, 1)); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); } }); // Test date matcher add_method("date", []() { { auto m = get_matcher("yearmin=2000"); core::Record matched; wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 1999; wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 2000; wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmax=2000"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 2001; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 2000; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmin=2000, yearmax=2010"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 1999; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 2011; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.datetime.min.year = matched.datetime.max.year = 2000; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.datetime.min.year = matched.datetime.max.year = 2005; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.datetime.min.year = matched.datetime.max.year = 2010; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } }); // Test coordinates matcher add_method("coords", []() { { auto m = get_matcher("latmin=45.00"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lat(43.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lat(45.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.station.coords.set_lat(46.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("latmax=45.00"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.lat = 4600000; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.lat = 4500000; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.station.coords.lat = 4400000; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=45.00, lonmax=180.0"); core::Record matched; wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(43.0); wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(45.0); wassert(actual(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=-180, lonmax=45.0"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(46.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(45.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); matched.station.coords.set_lon(44.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } { auto m = get_matcher("latmin=45.0, latmax=46.0, lonmin=10.0, lonmax=12.0"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lat(45.5); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(13.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.coords.set_lon(11.0); wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); } }); // Test rep_memo matcher add_method("rep_memo", []() { auto m = get_matcher("rep_memo=synop"); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.report = "temp"; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_NO); matched.station.report = "synop"; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); }); // Test empty matcher add_method("empty", []() { auto query = dballe::Query::create(); std::unique_ptr m = Matcher::create(*query); core::Record matched; wassert(actual_matcher_result(m->match(MatchedRecord(matched))) == matcher::MATCH_YES); }); #endif } } dballe-8.6/dballe/core/data-access.in.cc0000644000175000017500000000310613572425700014777 00000000000000#include "data.h" #include "var.h" #include using namespace wreport; namespace dballe { namespace core { void Data::setf(const char* key, unsigned len, const char* val) { switch (key) { // mklookup case "rep_memo": station.report = val; case "report": station.report = val; case "ana_id": station.id = strtol(val, nullptr, 10); case "ident": station.ident = val; case "lat": { double dval = strtod(val, nullptr); station.coords.set_lat(dval); } case "lon": { double dval = strtod(val, nullptr); station.coords.set_lon(dval); } case "year": datetime.year = strtol(val, nullptr, 10); case "month": datetime.month = strtol(val, nullptr, 10); case "day": datetime.day = strtol(val, nullptr, 10); case "hour": datetime.hour = strtol(val, nullptr, 10); case "min": datetime.minute = strtol(val, nullptr, 10); case "sec": datetime.second = strtol(val, nullptr, 10); case "leveltype1": level.ltype1 = strtol(val, nullptr, 10); case "l1": level.l1 = strtol(val, nullptr, 10); case "leveltype2": level.ltype2 = strtol(val, nullptr, 10); case "l2": level.l2 = strtol(val, nullptr, 10); case "pindicator": trange.pind = strtol(val, nullptr, 10); case "p1": trange.p1 = strtol(val, nullptr, 10); case "p2": trange.p2 = strtol(val, nullptr, 10); default: values.setf(key, val); } } } } dballe-8.6/dballe/core/csv.cc0000644000175000017500000003345213554564112013025 00000000000000/* * dballe/csv - CSV utility functions * * Copyright (C) 2005--2014 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ /* * The FormatInt class is Copyright (c) 2012, Victor Zverovich * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions are met: * * 1. Redistributions of source code must retain the above copyright notice, this * list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright notice, * this list of conditions and the following disclaimer in the documentation * and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ #include #include #include "dballe/var.h" #include #include #include #include #include #include #include #include #include using namespace std; using namespace wreport; namespace dballe { CSVReader::CSVReader() : in(0), close_on_exit(false) {} CSVReader::CSVReader(std::istream& in) : in(&in), close_on_exit(false) {} CSVReader::CSVReader(const std::string& pathname) : in(0), close_on_exit(false) { open(pathname); } CSVReader::~CSVReader() {} void CSVReader::open(const std::string& pathname) { close(); close_on_exit = true; in = new ifstream(pathname.c_str()); if (in->fail()) error_system::throwf("cannot open file %s", pathname.c_str()); } void CSVReader::close() { if (in && close_on_exit) delete in; in = 0; close_on_exit = true; } std::string CSVReader::unescape(const std::string& csvstr) { if (csvstr.empty()) return csvstr; if (csvstr[0] != '"') return csvstr; if (csvstr.size() == 1) return csvstr; string res; bool escape = false; for (string::const_iterator i = csvstr.begin() + 1; i != csvstr.end(); ++i) { if (*i == '"') { if (!escape) escape = true; else { res += *i; escape = false; } } else res += *i; } return res; } int CSVReader::as_int(unsigned col) const { if (cols[col].empty()) error_consistency::throwf("cannot parse a number from column %u, which is empty", col); return strtoul(cols[col].c_str(), 0, 10); } int CSVReader::as_int_withmissing(unsigned col) const { if (cols[col].empty()) return MISSING_INT; return strtoul(cols[col].c_str(), 0, 10); } wreport::Varcode CSVReader::as_varcode(unsigned col) const { return resolve_varcode(cols[col]); } bool CSVReader::move_to_data(unsigned number_col) { while (true) { if (number_col < cols.size() && (cols[number_col].empty() || isdigit(cols[number_col][0]) || cols[number_col][0] == '-')) break; if (!next()) return false; } return true; } int CSVReader::next_char() { int res = in->get(); if (res == EOF && !in->eof()) throw error_system("reading a character from CSV input"); return res; } bool CSVReader::next() { if (!in) return false; cols.clear(); // Tokenize the input line enum State { BEG, COL, QCOL, EQCOL, HALFEOL } state = BEG; string col; int c; while ((c = next_char()) != EOF) { switch (state) { // Look for the beginning of a column value case BEG: switch (c) { case '"': state = QCOL; break; case ',': state = BEG; cols.push_back(col); col.clear(); break; case '\r': state = HALFEOL; break; case '\n': cols.push_back(col); return true; default: state = COL; col += c; break; } break; // Inside a column value case COL: switch (c) { case ',': state = BEG; cols.push_back(col); col.clear(); break; case '\r': state = HALFEOL; break; case '\n': cols.push_back(col); return true; default: col += c; break; } break; // Inside a quoted column value case QCOL: switch (c) { case '\"': state = EQCOL; break; default: col += c; break; } break; // After a quote character found inside a quoted column value case EQCOL: switch (c) { // The quote marked the end of the value case ',': state = BEG; cols.push_back(col); col.clear(); break; case '\r': state = HALFEOL; break; case '\n': cols.push_back(col); return true; // The quote was an escape default: state = QCOL; col += c; break; } break; // After \r was found case HALFEOL: switch (c) { case '\n': cols.push_back(col); return true; default: state = COL; col += '\r'; col += c; break; } break; } } if (state == BEG) return false; if (!col.empty()) cols.push_back(col); return true; } bool csv_read_next(FILE* in, std::vector& cols) { char line[2000]; char* tok; char* stringp; if (fgets(line, 2000, in) == NULL) return false; cols.clear(); for (stringp = line; (tok = strsep(&stringp, ",")) != NULL; ) cols.push_back(tok); return true; } void csv_output_quoted_string(ostream& out, const std::string& str) { if (str.find_first_of("\",") != string::npos) { out << "\""; for (string::const_iterator i = str.begin(); i != str.end(); ++i) { if (*i == '"') out << '"'; out << *i; } out << "\""; } else out << str; } void csv_output_quoted_string(FILE* out, const std::string& str) { if (str.find_first_of("\",") != string::npos) { putc('"', out); for (string::const_iterator i = str.begin(); i != str.end(); ++i) { if (*i == '"') putc('"', out); putc(*i, out); } putc('"', out); } else fputs(str.c_str(), out); } namespace { // FormatInt by Victor Zverovich // See https://github.com/vitaut/format const char DIGITS[] = "0001020304050607080910111213141516171819" "2021222324252627282930313233343536373839" "4041424344454647484950515253545556575859" "6061626364656667686970717273747576777879" "8081828384858687888990919293949596979899"; class FormatInt { private: // Buffer should be large enough to hold all digits (digits10 + 1), // a sign and a null character. enum {BUFFER_SIZE = std::numeric_limits::digits10 + 3}; char buffer_[BUFFER_SIZE]; char *str_; // Formats value in reverse and returns the number of digits. char *FormatDecimal(uint64_t value) { char *buffer_end = buffer_ + BUFFER_SIZE; *--buffer_end = '\0'; while (value >= 100) { // Integer division is slow so do it for a group of two digits instead // of for every digit. The idea comes from the talk by Alexandrescu // "Three Optimization Tips for C++". See speed-test for a comparison. unsigned index = (value % 100) * 2; value /= 100; *--buffer_end = DIGITS[index + 1]; *--buffer_end = DIGITS[index]; } if (value < 10) { *--buffer_end = static_cast('0' + value); return buffer_end; } unsigned index = static_cast(value * 2); *--buffer_end = DIGITS[index + 1]; *--buffer_end = DIGITS[index]; return buffer_end; } public: explicit FormatInt(int value) { unsigned abs_value = value; bool negative = value < 0; if (negative) abs_value = 0 - value; str_ = FormatDecimal(abs_value); if (negative) *--str_ = '-'; } explicit FormatInt(unsigned value) : str_(FormatDecimal(value)) {} explicit FormatInt(uint64_t value) : str_(FormatDecimal(value)) {} unsigned size() const { return buffer_ + BUFFER_SIZE - str_ - 1; } const char *c_str() const { return str_; } std::string str() const { return str_; } }; } CSVWriter::~CSVWriter() { } void CSVWriter::add_value_empty() { if (!row.empty()) row += ','; } void CSVWriter::add_value_raw(const char* str) { if (!row.empty()) row += ','; row.append(str); } void CSVWriter::add_value_raw(const std::string& str) { if (!row.empty()) row += ','; row.append(str); } void CSVWriter::add_value(uint64_t val) { FormatInt fmt(val); add_value_raw(fmt.c_str()); } void CSVWriter::add_value(unsigned val) { FormatInt fmt(val); add_value_raw(fmt.c_str()); } void CSVWriter::add_value(int val) { FormatInt fmt(val); add_value_raw(fmt.c_str()); } void CSVWriter::add_value_withmissing(int val) { if (val == MISSING_INT) { if (!row.empty()) row += ','; } else { add_value(val); } } void CSVWriter::add_value(wreport::Varcode val) { if (!row.empty()) row += ','; switch (WR_VAR_F(val)) { case 0: row += 'B'; break; case 1: row += 'R'; break; case 2: row += 'C'; break; case 3: row += 'D'; break; } unsigned index = WR_VAR_X(val) * 2; row += DIGITS[index]; row += DIGITS[index + 1]; FormatInt fmt(WR_VAR_Y(val)); for (unsigned i = 0; i < 3-fmt.size(); ++i) row += '0'; row.append(fmt.c_str()); } void CSVWriter::add_var_value_raw(const wreport::Var& var) { if (!var.isset()) { add_value_empty(); return; } switch (var.info()->type) { case Vartype::String: add_value(var.enqc()); break; case Vartype::Binary: // Skip binary variables, that cannot really be encoded in CSV add_value_empty(); break; case Vartype::Integer: case Vartype::Decimal: add_value(var.enqi()); break; } } void CSVWriter::add_var_value_formatted(const wreport::Var& var) { if (!var.isset()) { add_value_empty(); return; } switch (var.info()->type) { case Vartype::String: add_value(var.enqc()); break; case Vartype::Binary: // Skip binary variables, that cannot really be encoded in CSV add_value_empty(); break; case Vartype::Integer: add_value(var.enqi()); break; case Vartype::Decimal: add_value_raw(var.format("")); break; } } void CSVWriter::add_value(const char* val) { if (!row.empty()) row += ','; if (!*val) return; row += '"'; for ( ; *val; ++val) { if (*val == '"') row += '"'; row += *val; } row += '"'; } void CSVWriter::add_value(const std::string& val) { if (!row.empty()) row += ','; if (val.empty()) return; row += '"'; for (string::const_iterator i = val.begin(); i != val.end(); ++i) { if (*i == '"') row += '"'; row += *i; } row += '"'; } } dballe-8.6/dballe/core/json.h0000644000175000017500000001541313554564112013042 00000000000000#ifndef DBALLE_CORE_JSON_H #define DBALLE_CORE_JSON_H #include #include #include #include #include #include #include namespace dballe { namespace core { struct JSONParseException : public std::runtime_error { using std::runtime_error::runtime_error; }; /** * JSON serializer * * It is called with a sequence of sax-like events, and appends the resulting * JSON to a string. * * The JSON output is all in one line, so that end of line can be used as * separator between distinct JSON records. */ class JSONWriter { protected: enum State { LIST_FIRST, LIST, MAPPING_KEY_FIRST, MAPPING_KEY, MAPPING_VAL, }; std::ostream& out; std::vector stack; /// Append whatever separator is needed (if any) before a new value void val_head(); void jputc(char c); void jputs(const char* s); public: JSONWriter(std::ostream& out); ~JSONWriter(); /** * Reset the serializer state, to cancel the current output and prepare for * a new one */ void reset(); void start_list(); void end_list(); void start_mapping(); void end_mapping(); void add_null(); void add_bool(bool val); void add_int(int val); void add_double(double val); void add_cstring(const char* val); void add_string(const std::string& val); template void add_ostream(const T& val) { val_head(); out << val; } void add_number(const std::string& val); void add_level(const Level& val); void add_trange(const Trange& val); void add_datetime(const Datetime& val); void add_datetimerange(const DatetimeRange& val); void add_coords(const Coords& val); void add_ident(const Ident& val); void add_var(const wreport::Var& val); void add_station(const Station& s); void add_dbstation(const DBStation& s); void add_values(const Values& values); void add_dbvalues(const DBValues& values); void add_break(); void add(const std::string& val) { add_string(val); } void add(const char* val) { add_cstring(val); } void add(double val) { add_double(val); } void add(int val) { add_int(val); } void add(bool val) { add_bool(val); } void add(size_t val) { add_ostream(val); } void add(wreport::Varcode val) { add_int(val); } void add(const Level& val) { add_level(val); } void add(const Trange& val) { add_trange(val); } void add(const Datetime& val) { add_datetime(val); } void add(const DatetimeRange& val) { add_datetimerange(val); } void add(const Coords& val) { add_coords(val); } void add(const Ident& val) { add_ident(val); } void add(const wreport::Var& val) { add_var(val); } void add(const Station& s) { add_station(s); } void add(const DBStation& s) { add_dbstation(s); } void add(const Values& v) { add_values(v); } void add(const DBValues& v) { add_dbvalues(v); } template void add(const char* a, T b) { add_cstring(a); add(b); } template void add_list(const T& val) { start_list(); for (const auto& i : val) add(i); end_list(); } }; /** * JSON sax-like parser. */ class JSONReader { public: virtual ~JSONReader() {} virtual void on_start_list() = 0; virtual void on_end_list() = 0; virtual void on_start_mapping() = 0; virtual void on_end_mapping() = 0; virtual void on_add_null() = 0; virtual void on_add_bool(bool val) = 0; virtual void on_add_int(int val) = 0; virtual void on_add_double(double val) = 0; virtual void on_add_string(const std::string& val) = 0; // Parse a stream void parse(std::istream& in); }; namespace json { enum Element { JSON_OBJECT, JSON_ARRAY, JSON_STRING, JSON_NUMBER, JSON_TRUE, JSON_FALSE, JSON_NULL, }; struct Stream { std::istream& in; Stream(std::istream& in) : in(in) {} /// Raise a parse error if the stream does not yield this exact token void expect_token(const char* token); /// Consume and discard all spaces at the start of the stream void skip_spaces(); /// Parse an unsigned integer template T parse_unsigned() { T res = 0; while (true) { int c = in.peek(); if (c >= '0' and c <= '9') res = res * 10 + in.get() - '0'; else break; } skip_spaces(); return res; } /// Parse a signed integer template T parse_signed() { if (in.peek() == '-') { in.get(); return -parse_unsigned(); } else return parse_unsigned(); } /// Parse a double double parse_double(); /** * Parse a number, without converting it. * * Returns the number, and true if it looks like a floating point number, * false if it looks like an integer number */ std::tuple parse_number(); /// Parse a string from the start of the stream std::string parse_string(); /// Parse a Coords object Coords parse_coords(); /// Parse a Coords object Station parse_station(); /// Parse a Coords object DBStation parse_dbstation(); /// Parse an Ident object Ident parse_ident(); /// Parse a Level object Level parse_level(); /// Parse a Trange object Trange parse_trange(); /// Parse a Datetime object Datetime parse_datetime(); /// Parse a DatetimeRange object DatetimeRange parse_datetimerange(); template inline T parse() { throw wreport::error_unimplemented(); } /// Parse a JSON array, calling on_element to parse each element void parse_array(std::function on_element); /// Parse a JSON object, calling on_value to parse each value void parse_object(std::function on_value); /// Identify the next element in the stream, without moving the stream /// position Element identify_next(); }; template<> inline std::string Stream::parse() { return parse_string(); } template<> inline Coords Stream::parse() { return parse_coords(); } template<> inline Station Stream::parse() { return parse_station(); } template<> inline DBStation Stream::parse() { return parse_dbstation(); } template<> inline Ident Stream::parse() { return parse_ident(); } template<> inline Level Stream::parse() { return parse_level(); } template<> inline Trange Stream::parse() { return parse_trange(); } template<> inline Datetime Stream::parse() { return parse_datetime(); } template<> inline DatetimeRange Stream::parse() { return parse_datetimerange(); } } } } #endif dballe-8.6/dballe/core/varmatch.h0000644000175000017500000000104113554564112013666 00000000000000#ifndef DBA_CORE_VARMATCH_H #define DBA_CORE_VARMATCH_H #include #include namespace dballe { /** * Match a variable code and value */ struct Varmatch { wreport::Varcode code; Varmatch(wreport::Varcode code); virtual ~Varmatch() {} virtual bool operator()(const wreport::Var&) const; /** * Parse variable matcher from a string in the form * Bxxyyy{<|<=|=|>=|>}value or value<=Bxxyyy<=value */ static std::unique_ptr parse(const std::string& filter); }; } #endif dballe-8.6/dballe/core/tests.h0000644000175000017500000000531413554564112013232 00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include namespace dballe { namespace tests { using namespace wreport::tests; #if 0 // Some utility random generator functions static inline int rnd(int min, int max) { return min + (int) ((max - min) * (rand() / (RAND_MAX + 1.0))); } static inline double rnd(double min, double max) { return min + (int) ((max - min) * (rand() / (RAND_MAX + 1.0))); } static inline std::string rnd(int len) { std::string res; int max = rnd(1, len); for (int i = 0; i < max; i++) res += (char)rnd('a', 'z'); return res; } static inline bool rnd(double prob) { return (rnd(0, 100) < prob*100) ? true : false; } #endif // Message reading functions /// Return the pathname of a test file std::string datafile(const std::string& fname); std::unique_ptr open_test_data(const char* filename, Encoding type); BinaryMessage read_rawmsg(const char* filename, Encoding type); class MemoryCSVWriter : public CSVWriter { public: std::stringstream buf; void flush_row() override { buf << row << std::endl; row.clear(); } }; #if 0 /// Check that actual and expected have the same vars struct TestRecordValEqual { const dballe::Record& actual; const dballe::Record& expected; const char* name; bool with_missing_int; TestRecordValEqual(const dballe::Record& actual, const dballe::Record& expected, const char* name, bool with_missing_int=false) : actual(actual), expected(expected), name(name), with_missing_int(with_missing_int) {} void check() const; }; struct TestRecordVarsEqual { const dballe::Record& actual; dballe::Values expected; TestRecordVarsEqual(const dballe::Record& actual, const dballe::Record& expected) : actual(actual), expected(expected) {} TestRecordVarsEqual(const dballe::Record& actual, const dballe::Values& expected) : actual(actual), expected(expected) {} void check() const; }; #endif // Set a query from a ", "-separated string of assignments std::unique_ptr query_from_string(const std::string& s); core::Query core_query_from_string(const std::string& s); struct ActualMatcherResult : public Actual { using Actual::Actual; void operator==(int expected) const; void operator!=(int expected) const; }; inline ActualMatcherResult actual_matcher_result(int actual) { return ActualMatcherResult(actual); } using wreport::tests::actual; inline ActualCString actual(const dballe::Ident& ident) { return ActualCString(ident); } } } dballe-8.6/dballe/core/aliases.cc0000644000175000017500000001313413554574016013652 00000000000000/* C++ code produced by gperf version 3.1 */ /* Command-line: gperf */ /* Computed positions: -k'$' */ #if !((' ' == 32) && ('!' == 33) && ('"' == 34) && ('#' == 35) \ && ('%' == 37) && ('&' == 38) && ('\'' == 39) && ('(' == 40) \ && (')' == 41) && ('*' == 42) && ('+' == 43) && (',' == 44) \ && ('-' == 45) && ('.' == 46) && ('/' == 47) && ('0' == 48) \ && ('1' == 49) && ('2' == 50) && ('3' == 51) && ('4' == 52) \ && ('5' == 53) && ('6' == 54) && ('7' == 55) && ('8' == 56) \ && ('9' == 57) && (':' == 58) && (';' == 59) && ('<' == 60) \ && ('=' == 61) && ('>' == 62) && ('?' == 63) && ('A' == 65) \ && ('B' == 66) && ('C' == 67) && ('D' == 68) && ('E' == 69) \ && ('F' == 70) && ('G' == 71) && ('H' == 72) && ('I' == 73) \ && ('J' == 74) && ('K' == 75) && ('L' == 76) && ('M' == 77) \ && ('N' == 78) && ('O' == 79) && ('P' == 80) && ('Q' == 81) \ && ('R' == 82) && ('S' == 83) && ('T' == 84) && ('U' == 85) \ && ('V' == 86) && ('W' == 87) && ('X' == 88) && ('Y' == 89) \ && ('Z' == 90) && ('[' == 91) && ('\\' == 92) && (']' == 93) \ && ('^' == 94) && ('_' == 95) && ('a' == 97) && ('b' == 98) \ && ('c' == 99) && ('d' == 100) && ('e' == 101) && ('f' == 102) \ && ('g' == 103) && ('h' == 104) && ('i' == 105) && ('j' == 106) \ && ('k' == 107) && ('l' == 108) && ('m' == 109) && ('n' == 110) \ && ('o' == 111) && ('p' == 112) && ('q' == 113) && ('r' == 114) \ && ('s' == 115) && ('t' == 116) && ('u' == 117) && ('v' == 118) \ && ('w' == 119) && ('x' == 120) && ('y' == 121) && ('z' == 122) \ && ('{' == 123) && ('|' == 124) && ('}' == 125) && ('~' == 126)) /* The character set is not based on ISO-646. */ #error "gperf generated tables don't work with this execution character set. Please report a bug to ." #endif #include #include using namespace wreport; namespace dballe { struct aliasdef { const char* alias; Varcode var; }; #define TOTAL_KEYWORDS 24 #define MIN_WORD_LENGTH 1 #define MAX_WORD_LENGTH 10 #define MIN_HASH_VALUE 1 #define MAX_HASH_VALUE 32 /* maximum key range = 32, duplicates = 0 */ class VarcodeAliases { private: static inline unsigned int hash (const char *str, size_t len); public: static struct aliasdef *find (const char *str, size_t len); }; inline unsigned int VarcodeAliases::hash (const char *str, size_t len) { static unsigned char asso_values[] = { 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 20, 33, 10, 30, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 5, 15, 10, 33, 20, 33, 33, 0, 33, 33, 20, 15, 0, 25, 33, 33, 5, 20, 15, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33, 33 }; return len + asso_values[static_cast(str[len - 1])]; } static struct aliasdef wordlist[] = { {""}, {"p", WR_VAR(0, 10, 4)}, {"tp", WR_VAR(0, 13, 11)}, {"mwp", WR_VAR(0, 22, 74)}, {"mslp", WR_VAR(0, 10, 51)}, {"block", WR_VAR(0, 1, 1)}, {"t", WR_VAR(0, 12, 101)}, {"td", WR_VAR(0, 12, 103)}, {"mwd", WR_VAR(0, 22, 1)}, {"pp1d", WR_VAR(0, 22, 71)}, {""}, {"height", WR_VAR(0, 7, 30)}, {"data_id", WR_VAR(0, 33, 195)}, {"no2", WR_VAR(0, 15, 193)}, {"conf", WR_VAR(0, 33, 7)}, {""}, {"v", WR_VAR(0, 11, 4)}, {"no", WR_VAR(0, 15, 192)}, {""}, {"name", WR_VAR(0, 1, 19)}, {""}, {"u", WR_VAR(0, 11, 3)}, {"rh", WR_VAR(0, 13, 3)}, {"swh", WR_VAR(0, 22, 70)}, {"pm10", WR_VAR(0, 15, 195)}, {"heightbaro", WR_VAR(0, 7, 31)}, {"q", WR_VAR(0, 13, 1)}, {"station", WR_VAR(0, 1, 2)}, {""}, {""}, {""}, {""}, {"o3", WR_VAR(0, 15, 194)} }; struct aliasdef * VarcodeAliases::find (const char *str, size_t len) { if (len <= MAX_WORD_LENGTH && len >= MIN_WORD_LENGTH) { unsigned int key = hash (str, len); if (key <= MAX_HASH_VALUE) { const char *s = wordlist[key].alias; if (*str == *s && !strncmp (str + 1, s + 1, len - 1) && s[len] == '\0') return &wordlist[key]; } } return 0; } Varcode varcode_alias_resolve(const char* alias) { struct aliasdef* res = VarcodeAliases::find(alias, strlen(alias)); if (res == NULL) return 0; else return res->var; } Varcode varcode_alias_resolve_substring(const char* alias, int len) { struct aliasdef* res = VarcodeAliases::find(alias, len); if (res == NULL) return 0; else return res->var; } wreport::Varcode varcode_alias_resolve(const std::string& alias) { struct aliasdef* res = VarcodeAliases::find(alias.data(), alias.size()); if (res == NULL) return 0; else return res->var; } } /* vim:set ts=4 sw=4: */ dballe-8.6/dballe/core/string.cc0000644000175000017500000000443513554564112013537 00000000000000#include "string.h" using namespace std; namespace dballe { namespace { struct QSParser { std::string& url; size_t qs_start = 0; size_t pos = 0; size_t value_start = 0; QSParser(std::string& url) : url(url) { } bool seek_qs() { pos = url.find('?'); if (pos == string::npos) return false; ++pos; qs_start = pos; return true; } bool next_arg() { // Move to the next query string argument pos = url.find('&', pos); if (pos == string::npos) return false; ++pos; return true; } bool value(const std::string& key) { if (url.substr(pos, key.size()) != key) return false; if (pos + key.size() == url.size()) { value_start = url.size(); return true; } if (url[pos + key.size()] == '=') { value_start = pos + key.size() + 1; return true; } if (url[pos + key.size()] == '&') { value_start = pos + key.size(); return true; } return false; } std::string pop_value() { std::string res; size_t value_end = url.find('&', value_start); if (value_end == string::npos) { res = url.substr(value_start); url.erase(pos); } else { res = url.substr(value_start, value_end - value_start); url.erase(pos, value_end - pos + 1); } // Remove trailing ? if there was only this value in the query string if (pos == url.size()) url.resize(url.size() - 1); return res; } }; } bool url_pop_query_string(std::string& url, const std::string& name, std::string& val) { QSParser parser(url); // Look for the beginning of the query string if (!parser.seek_qs()) return false; // Look for the beginning of the argument in the query string while (true) { if (!parser.value(name)) { // Move to the next query string argument if (!parser.next_arg()) return false; continue; } val = parser.pop_value(); return true; } } } dballe-8.6/dballe/core/defs.h0000644000175000017500000000043313554564112013006 00000000000000#ifndef DBA_MSG_DEFS_H #define DBA_MSG_DEFS_H /** @file * Common definitions */ #if defined(_DBALLE_TEST_CODE) || defined(_DBALLE_LIBRARY_CODE) #define DBALLE_TEST_ONLY #else #define DBALLE_TEST_ONLY [[deprecated("this is intended for DB-All.e unit tests only")]] #endif #endif dballe-8.6/dballe/core/cursor.h0000644000175000017500000000513113554564112013402 00000000000000#ifndef DBALLE_CORE_CURSOR_H #define DBALLE_CORE_CURSOR_H #include #include #include namespace dballe { namespace impl { /// Cursor iterating over stations struct CursorStation : public dballe::CursorStation { virtual void enq(Enq& enq) const = 0; /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { return std::unique_ptr(dynamic_cast(c.release())); } /// Create a CursorStation iterating on no results static std::unique_ptr make_empty(); }; /// Cursor iterating over station data values struct CursorStationData : public dballe::CursorStationData { virtual void enq(Enq& enq) const = 0; /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { return std::unique_ptr(dynamic_cast(c.release())); } /// Create a CursorStationData iterating on no results static std::unique_ptr make_empty(); }; /// Cursor iterating over data values struct CursorData : public dballe::CursorData { virtual void enq(Enq& enq) const = 0; /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { return std::unique_ptr(dynamic_cast(c.release())); } /// Create a CursorData iterating on no results static std::unique_ptr make_empty(); }; /// Cursor iterating over summary entries struct CursorSummary : public dballe::CursorSummary { virtual void enq(Enq& enq) const = 0; /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { return std::unique_ptr(dynamic_cast(c.release())); } /// Create a CursorSummary iterating on no results static std::unique_ptr make_empty(); }; /// Cursor iterating over messages struct CursorMessage : public dballe::CursorMessage { virtual void enq(Enq& enq) const {} /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { return std::unique_ptr(dynamic_cast(c.release())); } /// Create a CursorStation iterating on no results static std::unique_ptr make_empty(); }; } } #endif dballe-8.6/dballe/core/data-test.cc0000644000175000017500000000525413554564112014117 00000000000000#include "tests.h" #include "data.h" using namespace dballe::tests; using namespace dballe; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_data"); void Tests::register_tests() { #if 0 add_method("comparisons", []() { core::Record rec; rec.station.id = -10; rec.station.coords.set(12.34567, 76.54321); rec.datetime.min.year = 1976; rec.obtain(WR_VAR(0, 20, 1)).set("456"); rec.obtain(WR_VAR(0, 20, 3)).set("456"); core::Record rec1; rec1 = rec; wassert(actual(rec == rec1).istrue()); wassert(actual(rec1 == rec).istrue()); wassert(actual(!(rec != rec1)).istrue()); wassert(actual(!(rec1 != rec)).istrue()); rec1.datetime.min.year = 1975; wassert(actual(rec != rec1).istrue()); wassert(actual(rec1 != rec).istrue()); wassert(actual(!(rec == rec1)).istrue()); wassert(actual(!(rec1 == rec)).istrue()); rec1 = rec; wassert(actual(rec == rec1).istrue()); wassert(actual(rec1 == rec).istrue()); rec1.datetime = DatetimeRange(); wassert(actual(rec != rec1).istrue()); wassert(actual(rec1 != rec).istrue()); rec1 = rec; wassert(actual(rec == rec1).istrue()); wassert(actual(rec1 == rec).istrue()); rec1.obtain(WR_VAR(0, 20, 1)).set("45"); wassert(actual(rec != rec1).istrue()); wassert(actual(rec1 != rec).istrue()); rec1 = rec; wassert(actual(rec == rec1).istrue()); wassert(actual(rec1 == rec).istrue()); rec1.unset_var(WR_VAR(0, 20, 1)); wassert(actual(rec != rec1).istrue()); wassert(actual(rec1 != rec).istrue()); }); add_method("extremes", []() { // Test querying extremes by Datetime core::Record rec; DatetimeRange dtr = wcallchecked(rec.get_datetimerange()); wassert(actual(dtr.min.is_missing()).istrue()); wassert(actual(dtr.max.is_missing()).istrue()); static const int NA = MISSING_INT; wassert(rec.set(DatetimeRange(2010, 1, 1, 0, 0, 0, NA, NA, NA, NA, NA, NA))); dtr = wcallchecked(rec.get_datetimerange()); wassert(actual(dtr.min) == Datetime(2010, 1, 1, 0, 0, 0)); wassert(actual(dtr.max.is_missing()).istrue()); wassert(rec.set(DatetimeRange(NA, NA, NA, NA, NA, NA, 2011, 2, 3, 4, 5, 6))); dtr = wcallchecked(rec.get_datetimerange()); wassert(actual(dtr.min.is_missing()).istrue()); wassert(actual(dtr.max) == Datetime(2011, 2, 3, 4, 5, 6)); wassert(rec.set(DatetimeRange(2010, 1, 1, 0, 0, 0, 2011, 2, 3, 4, 5, 6))); dtr = wcallchecked(rec.get_datetimerange()); wassert(actual(dtr.min) == Datetime(2010, 1, 1, 0, 0, 0)); wassert(actual(dtr.max) == Datetime(2011, 2, 3, 4, 5, 6)); }); #endif } } dballe-8.6/dballe/core/smallset.h0000644000175000017500000001610413554564112013713 00000000000000#ifndef DBALLE_CORE_SMALLSET_H #define DBALLE_CORE_SMALLSET_H #include #include namespace dballe { namespace core { template inline const T& smallset_default_get_value(const T& val) { return val; } /** * Set structure optimized for a small number of items */ template struct SmallSet { mutable std::vector items; mutable size_t dirty = 0; typedef typename std::vector::const_iterator const_iterator; typedef typename std::vector::iterator iterator; typedef typename std::vector::const_reverse_iterator const_reverse_iterator; typedef typename std::vector::reverse_iterator reverse_iterator; iterator begin() { return items.begin(); } iterator end() { return items.end(); } const_iterator begin() const { return items.begin(); } const_iterator end() const { return items.end(); } reverse_iterator rbegin() { return items.rbegin(); } reverse_iterator rend() { return items.rend(); } const_reverse_iterator rbegin() const { return items.rbegin(); } const_reverse_iterator rend() const { return items.rend(); } size_t size() const { return items.size(); } bool empty() const { return items.empty(); } bool operator==(const SmallSet& o) const { if (dirty) rearrange_dirty(); if (o.dirty) o.rearrange_dirty(); return items == o.items; } bool operator!=(const SmallSet& o) const { if (dirty) rearrange_dirty(); if (o.dirty) o.rearrange_dirty(); return items == o.items; } void clear() { items.clear(); dirty = 0; } int binary_search(const Value& value) const { int begin, end; begin = -1, end = items.size(); while (end - begin > 1) { int cur = (end + begin) / 2; if (value < get_value(items[cur])) end = cur; else begin = cur; } if (begin == -1 || get_value(items[begin]) != value) return -1; else return begin; } const_iterator find(const Value& value) const { if (items.empty()) return end(); // Stick to linear search if the vector size is small if (items.size() < 6) { for (auto it = std::begin(items); it != std::end(items); ++it) if (get_value(*it) == value) return it; return end(); } // Use binary search for larger vectors if (dirty > 16) { std::sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return get_value(a) < get_value(b); }); dirty = 0; } else if (dirty) { // Use insertion sort, if less than 16 new elements appeared since the // last sort rearrange_dirty(); } int pos = binary_search(value); if (pos == -1) return items.end(); else return items.begin() + pos; } iterator find(const Value& value) { if (items.empty()) return end(); // Stick to linear search if the vector size is small if (items.size() < 6) { for (auto it = std::begin(items); it != std::end(items); ++it) if (get_value(*it) == value) return it; return end(); } // Use binary search for larger vectors if (dirty > 16) { std::sort(items.begin(), items.end(), [](const Item& a, const Item& b) { return get_value(a) < get_value(b); }); dirty = 0; } else if (dirty) { // Use insertion sort, if less than 16 new elements appeared since the // last sort rearrange_dirty(); } int pos = binary_search(value); if (pos == -1) return items.end(); else return items.begin() + pos; } Item& add(const Item& item) { ++dirty; items.emplace_back(item); return items.back(); } // static const Value& _smallset_get_value(const Item&); void rearrange_dirty() const { // Rearrange newly inserted items by insertion sort for (size_t i = items.size() - dirty; i < items.size(); ++i) for (size_t j = i; j > 0 && get_value(items[j]) < get_value(items[j - 1]); --j) std::swap(items[j], items[j - 1]); dirty = 0; } }; template struct SmallUniqueValueSet : protected SmallSet { using SmallSet::iterator; using SmallSet::const_iterator; using SmallSet::reverse_iterator; using SmallSet::const_reverse_iterator; using SmallSet::begin; using SmallSet::end; using SmallSet::rbegin; using SmallSet::rend; using SmallSet::empty; using SmallSet::size; using SmallSet::clear; bool operator==(const SmallUniqueValueSet& o) const { return SmallSet::operator==(o); } bool operator!=(const SmallUniqueValueSet& o) const { return SmallSet::operator!=(o); } void add(const Value& val) { auto i = this->find(val); if (i != this->end()) return; SmallSet::add(val); } bool has(const Value& val) const { return this->find(val) != this->end(); } }; template struct SortedSmallUniqueValueSet : public SmallUniqueValueSet { typedef typename SmallUniqueValueSet::iterator iterator; typedef typename SmallUniqueValueSet::const_iterator const_iterator; typedef typename SmallUniqueValueSet::reverse_iterator reverse_iterator; typedef typename SmallUniqueValueSet::const_reverse_iterator const_reverse_iterator; using SmallUniqueValueSet::end; using SmallUniqueValueSet::rend; using SmallUniqueValueSet::empty; using SmallUniqueValueSet::size; using SmallUniqueValueSet::clear; using SmallUniqueValueSet::operator==; using SmallUniqueValueSet::operator!=; iterator begin() { if (this->dirty) this->rearrange_dirty(); return SmallUniqueValueSet::begin(); } const_iterator begin() const { if (this->dirty) this->rearrange_dirty(); return SmallUniqueValueSet::begin(); } reverse_iterator rbegin() { if (this->dirty) this->rearrange_dirty(); return SmallUniqueValueSet::rbegin(); } const_reverse_iterator rbegin() const { if (this->dirty) this->rearrange_dirty(); return SmallUniqueValueSet::rbegin(); } void add(const Value& val) { auto i = this->find(val); if (i != this->end()) return; SmallUniqueValueSet::add(val); } bool has(const Value& val) const { return this->find(val) != this->end(); } }; } } #endif dballe-8.6/dballe/core/query-access.cc0000644000175000017500000013157313572425274014646 00000000000000#include "query.h" #include "var.h" #include using namespace wreport; namespace dballe { namespace core { void Query::setf(const char* key, unsigned len, const char* val) { switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': level.l1 = strtol(val, nullptr, 10); break; case '2': level.l2 = strtol(val, nullptr, 10); break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'p': switch (key[1]) { case '1': trange.p1 = strtol(val, nullptr, 10); break; case '2': trange.p2 = strtol(val, nullptr, 10); break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { { double dval = strtod(val, nullptr); latrange.set(dval, dval); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'o': if (key[2] == 'n') { { double dval = strtod(val, nullptr); lonrange.set(dval, dval); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'd': if (memcmp(key + 1, "ay", 2) == 0) { dtrange.min.day = dtrange.max.day = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "in", 2) == 0) { dtrange.min.minute = dtrange.max.minute = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "ec", 2) == 0) { dtrange.min.second = dtrange.max.second = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { varcodes.clear(); varcodes.insert(resolve_varcode(val)); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { dtrange.min.year = dtrange.max.year = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { dtrange.min.hour = dtrange.max.hour = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { ident = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { dtrange.min.month = dtrange.max.month = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'q': if (memcmp(key + 1, "uery", 4) == 0) { query = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'l': if (memcmp(key + 1, "imit", 4) == 0) { limit = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'b': if (memcmp(key + 1, "lock", 4) == 0) { block = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { report = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { ana_id = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { mobile = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'l': switch (key[1]) { case 'a': if (memcmp(key + 2, "tm", 2) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { latrange.imax = Coords::lat_to_int(strtod(val, nullptr)); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { latrange.imin = Coords::lat_to_int(strtod(val, nullptr)); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'o': if (memcmp(key + 2, "nm", 2) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { lonrange.imax = Coords::lon_to_int(strtod(val, nullptr)); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { lonrange.imin = Coords::lon_to_int(strtod(val, nullptr)); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'd': if (memcmp(key + 1, "aym", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { dtrange.max.day = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { dtrange.min.day = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "ecm", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { dtrange.max.second = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { dtrange.min.second = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 7: switch (key[0]) { case 'p': if (memcmp(key + 1, "riom", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { priomax = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { priomin = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'y': if (memcmp(key + 1, "earm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.year = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.year = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'h': if (memcmp(key + 1, "ourm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.hour = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.hour = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "inum", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.minute = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.minute = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'v': if (memcmp(key + 1, "arlist", 6) == 0) { varcodes.clear(); resolve_varlist(val, varcodes); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "tation", 6) == 0) { station = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { priomin = priomax = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { report = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "onthm", 5) == 0) { switch (key[6]) { case 'a': if (key[7] == 'x') { dtrange.max.month = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[7] == 'n') { dtrange.min.month = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': level.ltype1 = strtol(val, nullptr, 10); break; case '2': level.ltype2 = strtol(val, nullptr, 10); break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { trange.pind = strtol(val, nullptr, 10); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "na_filter", 9) == 0) { ana_filter = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 11: switch (key[0]) { case 'd': if (memcmp(key + 1, "ata_filter", 10) == 0) { data_filter = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "ttr_filter", 10) == 0) { attr_filter = val; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } void Query::unset(const char* key, unsigned len) { switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': level.l1 = MISSING_INT; break; case '2': level.l2 = MISSING_INT; break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'p': switch (key[1]) { case '1': trange.p1 = MISSING_INT; break; case '2': trange.p2 = MISSING_INT; break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { latrange = LatRange(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'o': if (key[2] == 'n') { lonrange = LonRange(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'd': if (memcmp(key + 1, "ay", 2) == 0) { dtrange.min.day = dtrange.max.day = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "in", 2) == 0) { dtrange.min.minute = dtrange.max.minute = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "ec", 2) == 0) { dtrange.min.second = dtrange.max.second = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { varcodes.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { dtrange.min.year = dtrange.max.year = 0xffff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { dtrange.min.hour = dtrange.max.hour = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { ident.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { dtrange.min.month = dtrange.max.month = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'q': if (memcmp(key + 1, "uery", 4) == 0) { query.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'l': if (memcmp(key + 1, "imit", 4) == 0) { limit = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'b': if (memcmp(key + 1, "lock", 4) == 0) { block = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { report.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { ana_id = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { mobile = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'l': switch (key[1]) { case 'a': if (memcmp(key + 2, "tm", 2) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { latrange.imax = LatRange::IMAX; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { latrange.imin = LatRange::IMIN; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'o': if (memcmp(key + 2, "nm", 2) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { lonrange.imax = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { lonrange.imin = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'd': if (memcmp(key + 1, "aym", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { dtrange.max.day = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { dtrange.min.day = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "ecm", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { dtrange.max.second = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[5] == 'n') { dtrange.min.second = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 7: switch (key[0]) { case 'p': if (memcmp(key + 1, "riom", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { priomax = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { priomin = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'y': if (memcmp(key + 1, "earm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.year = 0xffff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.year = 0xffff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'h': if (memcmp(key + 1, "ourm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.hour = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.hour = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "inum", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { dtrange.max.minute = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[6] == 'n') { dtrange.min.minute = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'v': if (memcmp(key + 1, "arlist", 6) == 0) { varcodes.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 's': if (memcmp(key + 1, "tation", 6) == 0) { station = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { priomin = priomax = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { report.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'm': if (memcmp(key + 1, "onthm", 5) == 0) { switch (key[6]) { case 'a': if (key[7] == 'x') { dtrange.max.month = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'i': if (key[7] == 'n') { dtrange.min.month = 0xff; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': level.ltype1 = MISSING_INT; break; case '2': level.ltype2 = MISSING_INT; break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { trange.pind = MISSING_INT; } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "na_filter", 9) == 0) { ana_filter.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 11: switch (key[0]) { case 'd': if (memcmp(key + 1, "ata_filter", 10) == 0) { data_filter.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; case 'a': if (memcmp(key + 1, "ttr_filter", 10) == 0) { attr_filter.clear(); } else { wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } break; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } } } dballe-8.6/dballe/core/data-access.cc0000644000175000017500000001427413572426662014412 00000000000000#include "data.h" #include "var.h" #include using namespace wreport; namespace dballe { namespace core { void Data::setf(const char* key, unsigned len, const char* val) { switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': level.l1 = strtol(val, nullptr, 10); break; case '2': level.l2 = strtol(val, nullptr, 10); break; default: values.setf(key, val); } break; case 'p': switch (key[1]) { case '1': trange.p1 = strtol(val, nullptr, 10); break; case '2': trange.p2 = strtol(val, nullptr, 10); break; default: values.setf(key, val); } break; default: values.setf(key, val); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { { double dval = strtod(val, nullptr); station.coords.set_lat(dval); } } else { values.setf(key, val); } break; case 'o': if (key[2] == 'n') { { double dval = strtod(val, nullptr); station.coords.set_lon(dval); } } else { values.setf(key, val); } break; default: values.setf(key, val); } break; case 'd': if (memcmp(key + 1, "ay", 2) == 0) { datetime.day = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; case 'm': if (memcmp(key + 1, "in", 2) == 0) { datetime.minute = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; case 's': if (memcmp(key + 1, "ec", 2) == 0) { datetime.second = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; default: values.setf(key, val); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { datetime.year = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { datetime.hour = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; default: values.setf(key, val); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { station.ident = val; } else { values.setf(key, val); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { datetime.month = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; default: values.setf(key, val); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { station.report = val; } else { values.setf(key, val); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { station.id = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; default: values.setf(key, val); } break; case 8: if (memcmp(key + 0, "rep_memo", 8) == 0) { station.report = val; } else { values.setf(key, val); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': level.ltype1 = strtol(val, nullptr, 10); break; case '2': level.ltype2 = strtol(val, nullptr, 10); break; default: values.setf(key, val); } } else { values.setf(key, val); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { trange.pind = strtol(val, nullptr, 10); } else { values.setf(key, val); } break; default: values.setf(key, val); } break; default: values.setf(key, val); } } } } dballe-8.6/dballe/core/cursor-test.cc0000644000175000017500000000046613554564112014523 00000000000000#include "dballe/core/tests.h" #include using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("core_cursor"); void Tests::register_tests() { add_method("empty", []{ }); } } dballe-8.6/dballe/core/shortcuts.cc0000644000175000017500000002032113554564124014262 00000000000000#include "shortcuts.h" #include "var.h" #include #include namespace dballe { namespace impl { const Shortcut& Shortcut::by_name(const char* name) { return by_name(name, strlen(name)); } const Shortcut& Shortcut::by_name(const std::string& name) { return by_name(name.data(), name.size()); } std::ostream& operator<<(std::ostream& out, const Shortcut& shortcut) { char buf[7]; format_bcode(shortcut.code, buf); return out << shortcut.level << ":" << shortcut.trange << ":" << buf; } namespace sc { const Shortcut st_type = { true, Level(), Trange(), WR_VAR(0, 2, 1) }; const Shortcut st_name = { true, Level(), Trange(), WR_VAR(0, 1, 19) }; const Shortcut st_name_icao = { true, Level(), Trange(), WR_VAR(0, 1, 63) }; const Shortcut rep_memo = { true, Level(), Trange(), WR_VAR(0, 1, 194) }; const Shortcut poll_lcode = { true, Level(), Trange(), WR_VAR(0, 1, 212) }; const Shortcut poll_scode = { true, Level(), Trange(), WR_VAR(0, 1, 213) }; const Shortcut poll_gemscode = { true, Level(), Trange(), WR_VAR(0, 1, 214) }; const Shortcut poll_source = { true, Level(), Trange(), WR_VAR(0, 1, 215) }; const Shortcut poll_atype = { true, Level(), Trange(), WR_VAR(0, 1, 216) }; const Shortcut poll_ttype = { true, Level(), Trange(), WR_VAR(0, 1, 217) }; const Shortcut flight_reg_no = { true, Level(), Trange(), WR_VAR(0, 1, 8) }; const Shortcut flight_phase = { true, Level(), Trange(), WR_VAR(0, 8, 4) }; const Shortcut flight_roll = { true, Level(), Trange(), WR_VAR(0, 2, 63) }; const Shortcut navsys = { true, Level(), Trange(), WR_VAR(0, 2, 61) }; const Shortcut data_relay = { true, Level(), Trange(), WR_VAR(0, 2, 62) }; const Shortcut wind_inst = { true, Level(), Trange(), WR_VAR(0, 2, 2) }; const Shortcut temp_precision = { true, Level(), Trange(), WR_VAR(0, 2, 5) }; const Shortcut latlon_spec = { true, Level(), Trange(), WR_VAR(0, 2, 70) }; const Shortcut timesig = { true, Level(), Trange(), WR_VAR(0, 8, 21) }; const Shortcut block = { true, Level(), Trange(), WR_VAR(0, 1, 1) }; const Shortcut station = { true, Level(), Trange(), WR_VAR(0, 1, 2) }; const Shortcut ident = { true, Level(), Trange(), WR_VAR(0, 1, 11) }; const Shortcut year = { true, Level(), Trange(), WR_VAR(0, 4, 1) }; const Shortcut month = { true, Level(), Trange(), WR_VAR(0, 4, 2) }; const Shortcut day = { true, Level(), Trange(), WR_VAR(0, 4, 3) }; const Shortcut hour = { true, Level(), Trange(), WR_VAR(0, 4, 4) }; const Shortcut minute = { true, Level(), Trange(), WR_VAR(0, 4, 5) }; const Shortcut second = { true, Level(), Trange(), WR_VAR(0, 4, 6) }; const Shortcut latitude = { true, Level(), Trange(), WR_VAR(0, 5, 1) }; const Shortcut longitude = { true, Level(), Trange(), WR_VAR(0, 6, 1) }; const Shortcut height_station = { true, Level(), Trange(), WR_VAR(0, 7, 30) }; const Shortcut height_baro = { true, Level(), Trange(), WR_VAR(0, 7, 31) }; const Shortcut height_release = { true, Level(), Trange(), WR_VAR(0, 7, 7) }; const Shortcut station_height_quality = { true, Level(), Trange(), WR_VAR(0, 33, 24) }; const Shortcut isobaric_surface = { true, Level(), Trange(), WR_VAR(0, 7, 4) }; const Shortcut st_dir = { false, Level(1), Trange(254,0,0), WR_VAR(0, 1, 12) }; const Shortcut st_speed = { false, Level(1), Trange(254,0,0), WR_VAR(0, 1, 13) }; const Shortcut meas_equip_type = { false, Level(1), Trange(254,0,0), WR_VAR(0, 2, 3) }; const Shortcut sonde_type = { false, Level(1), Trange(254,0,0), WR_VAR(0, 2, 11) }; const Shortcut sonde_method = { false, Level(1), Trange(254,0,0), WR_VAR(0, 2, 12) }; const Shortcut sonde_correction = { false, Level(1), Trange(254,0,0), WR_VAR(0, 2, 13) }; const Shortcut sonde_tracking = { false, Level(1), Trange(254,0,0), WR_VAR(0, 2, 14) }; const Shortcut press = { false, Level(1), Trange(254,0,0), WR_VAR(0, 10, 4) }; const Shortcut press_3h = { false, Level(1), Trange(4,0,10800), WR_VAR(0, 10, 60) }; const Shortcut press_24h = { false, Level(1), Trange(4,0,86400), WR_VAR(0, 10, 60) }; const Shortcut water_temp = { false, Level(1), Trange(254,0,0), WR_VAR(0, 22, 43) }; const Shortcut height_anem = { false, Level(1), Trange(254,0,0), WR_VAR(0, 10, 197) }; const Shortcut press_tend = { false, Level(1), Trange(205,0,10800), WR_VAR(0, 10, 63) }; const Shortcut visibility = { false, Level(1), Trange(254,0,0), WR_VAR(0, 20, 1) }; const Shortcut pres_wtr = { false, Level(1), Trange(254,0,0), WR_VAR(0, 20, 3) }; const Shortcut past_wtr1_3h = { false, Level(1), Trange(205,0,10800), WR_VAR(0, 20, 4) }; const Shortcut past_wtr1_6h = { false, Level(1), Trange(205,0,21600), WR_VAR(0, 20, 4) }; const Shortcut past_wtr2_3h = { false, Level(1), Trange(205,0,10800), WR_VAR(0, 20, 5) }; const Shortcut past_wtr2_6h = { false, Level(1), Trange(205,0,21600), WR_VAR(0, 20, 5) }; const Shortcut metar_wtr = { false, Level(1), Trange(254,0,0), WR_VAR(0, 20, 9) }; const Shortcut tot_prec1 = { false, Level(1), Trange(1,0,3600), WR_VAR(0, 13, 11) }; const Shortcut tot_prec3 = { false, Level(1), Trange(1,0,10800), WR_VAR(0, 13, 11) }; const Shortcut tot_prec6 = { false, Level(1), Trange(1,0,21600), WR_VAR(0, 13, 11) }; const Shortcut tot_prec12 = { false, Level(1), Trange(1,0,43200), WR_VAR(0, 13, 11) }; const Shortcut tot_prec24 = { false, Level(1), Trange(1,0,86400), WR_VAR(0, 13, 11) }; const Shortcut tot_snow = { false, Level(1), Trange(254,0,0), WR_VAR(0, 13, 13) }; const Shortcut state_ground = { false, Level(1), Trange(254,0,0), WR_VAR(0, 20, 62) }; const Shortcut press_msl = { false, Level(101), Trange(254,0,0), WR_VAR(0, 10, 51) }; const Shortcut qnh = { false, Level(103,2000), Trange(254,0,0), WR_VAR(0, 10, 52) }; const Shortcut temp_2m = { false, Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 101) }; const Shortcut wet_temp_2m = { false, Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 102) }; const Shortcut dewpoint_2m = { false, Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 103) }; const Shortcut humidity = { false, Level(103,2000), Trange(254,0,0), WR_VAR(0, 13, 3) }; const Shortcut wind_dir = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 1) }; const Shortcut wind_speed = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 2) }; const Shortcut wind_gust_max_speed = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 41) }; const Shortcut wind_gust_max_dir = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 43) }; const Shortcut ex_ccw_wind = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 16) }; const Shortcut ex_cw_wind = { false, Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 17) }; const Shortcut cloud_n = { false, Level(256), Trange(254,0,0), WR_VAR(0, 20, 10) }; const Shortcut cloud_nh = { false, Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 11) }; const Shortcut cloud_hh = { false, Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 13) }; const Shortcut cloud_cl = { false, Level(256,MISSING_INT,258,1), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_cm = { false, Level(256,MISSING_INT,258,2), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_ch = { false, Level(256,MISSING_INT,258,3), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_n1 = { false, Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 11) }; const Shortcut cloud_c1 = { false, Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_h1 = { false, Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 13) }; const Shortcut cloud_n2 = { false, Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 11) }; const Shortcut cloud_c2 = { false, Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_h2 = { false, Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 13) }; const Shortcut cloud_n3 = { false, Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 11) }; const Shortcut cloud_c3 = { false, Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_h3 = { false, Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 13) }; const Shortcut cloud_n4 = { false, Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 11) }; const Shortcut cloud_c4 = { false, Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 12) }; const Shortcut cloud_h4 = { false, Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 13) }; } } } dballe-8.6/dballe/core/csv.h0000644000175000017500000001155013554564112012662 00000000000000/* * dballe/csv - CSV utility functions * * Copyright (C) 2005--2014 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ #ifndef DBA_CSV_H #define DBA_CSV_H /** @file * @ingroup core * Routines to parse data in CSV format */ #include #include #include #include #include namespace dballe { /** * Parse a CSV line. * * @param in * The file where to read from * @param cols * The parsed columns. * @return * true if a new line was found, else false */ bool csv_read_next(FILE* in, std::vector& cols); class CSVReader { protected: std::istream* in; int next_char(); public: /** * If true, the input stream will be deleted upon destruction. * If false, it will be left alone. */ bool close_on_exit; /// Last line read std::string line; /// Parsed CSV columns for the last line read std::vector cols; CSVReader(); CSVReader(std::istream& in); CSVReader(const std::string& pathname); ~CSVReader(); /** * Open the given file and sets close_on_exit to true */ void open(const std::string& pathname); /** * Sets in to 0. * If close_on_exit is true, close the currently opened file. */ void close(); /** * Return the given column, as an integer. * * A missing value raises an exception. */ int as_int(unsigned col) const; /** * Return the given column, as an integer. * * A missing value is returned as MISSING_INT. */ int as_int_withmissing(unsigned col) const; /** * Return the given column, as a Varcode. * * A missing value raises an exception. */ wreport::Varcode as_varcode(unsigned col) const; /** * Find the first line where the given column exists and starts with a * number. * * This can be used to skip titles and empty lines, moving to the start of * the real data. Real data is identified by using a column that starts * with text in the headers and number in the data. * * @returns true if a data line has been found, false if we reached EOF */ bool move_to_data(unsigned number_col=0); /// Read the next CSV line, returning false if EOF is reached bool next(); static std::string unescape(const std::string& csvstr); }; // TODO: CSV readers allowing to peek on the next line without consuming it, to // allow the Msg parser to stop at msg boundary after peeking at a line // also, stripping newlines at end of lines // also, reading from istream // also, de-escaping strings (if they start with quote) /** * Output a string value, quoted if needed according to CSV rules */ void csv_output_quoted_string(std::ostream& out, const std::string& str); /** * Output a string value, quoted if needed according to CSV rules */ void csv_output_quoted_string(FILE* out, const std::string& str); class CSVWriter { protected: std::string row; public: virtual ~CSVWriter(); /// Add an empty value to the current row void add_value_empty(); /// Add a value to the current row, without any escaping void add_value_raw(const char* str); /// Add a value to the current row, without any escaping void add_value_raw(const std::string& str); /// Add an int value to the current row void add_value(int val); /// Add an int value that can potentially be missing void add_value_withmissing(int val); /// Add an int value to the current row void add_value(unsigned val); /// Add an int value to the current row void add_value(uint64_t val); /// Add a formatted varcode to the current row void add_value(wreport::Varcode val); /// Add a variable value, in its raw integer form void add_var_value_raw(const wreport::Var& val); /// Add a variable value, formatted void add_var_value_formatted(const wreport::Var& val); /// Add a string to the current row void add_value(const char* val); /// Add a string to the current row void add_value(const std::string& val); /// Write the current line to the output file, and start a new one virtual void flush_row() = 0; }; } #endif dballe-8.6/dballe/core/values.h0000644000175000017500000000155613554564112013373 00000000000000#ifndef DBALLE_CORE_VALUES_H #define DBALLE_CORE_VALUES_H #include #include #include namespace dballe { namespace core { namespace value { struct Encoder { std::vector buf; Encoder(); void append_uint16(uint16_t val); void append_uint32(uint32_t val); void append_cstring(const char* val); void append(const wreport::Var& var); void append_attributes(const wreport::Var& var); }; struct Decoder { const uint8_t* buf; unsigned size; Decoder(const std::vector& buf); uint16_t decode_uint16(); uint32_t decode_uint32(); const char* decode_cstring(); std::unique_ptr decode_var(); /** * Decode the attributes of var from a buffer */ static void decode_attrs(const std::vector& buf, wreport::Var& var); }; } } } #endif dballe-8.6/dballe/core/csv-test.cc0000644000175000017500000001645213554564112014003 00000000000000#include "tests.h" #include "csv.h" #include #include using namespace std; using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { // Test CSV string escaping add_method("escape", []() { stringstream s; csv_output_quoted_string(s, ""); wassert(actual(s.str()) == ""); wassert(actual(CSVReader::unescape(s.str())) == ""); s.str(std::string()); csv_output_quoted_string(s, "1"); wassert(actual(s.str()) == "1"); wassert(actual(CSVReader::unescape(s.str())) == "1"); s.str(std::string()); csv_output_quoted_string(s, "12"); wassert(actual(s.str()) == "12"); wassert(actual(CSVReader::unescape(s.str())) == "12"); s.str(std::string()); csv_output_quoted_string(s, "123"); wassert(actual(s.str()) == "123"); wassert(actual(CSVReader::unescape(s.str())) == "123"); s.str(std::string()); csv_output_quoted_string(s, ","); wassert(actual(s.str()) == "\",\""); wassert(actual(CSVReader::unescape(s.str())) == ","); s.str(std::string()); csv_output_quoted_string(s, "antani, blinda"); wassert(actual(s.str()) == "\"antani, blinda\""); wassert(actual(CSVReader::unescape(s.str())) == "antani, blinda"); s.str(std::string()); csv_output_quoted_string(s, "antani, \"blinda\""); wassert(actual(s.str()) == "\"antani, \"\"blinda\"\"\""); wassert(actual(CSVReader::unescape(s.str())) == "antani, \"blinda\""); s.str(std::string()); csv_output_quoted_string(s, "\""); wassert(actual(s.str()) == "\"\"\"\""); wassert(actual(CSVReader::unescape(s.str())) == "\""); s.str(std::string()); csv_output_quoted_string(s, "\",\""); wassert(actual(s.str()) == "\"\"\",\"\"\""); wassert(actual(CSVReader::unescape(s.str())) == "\",\""); wassert(actual(CSVReader::unescape("\"")) == "\""); wassert(actual(CSVReader::unescape("\"\"")) == ""); wassert(actual(CSVReader::unescape("\"\"\"")) == "\""); wassert(actual(CSVReader::unescape("\"\"\"\"")) == "\""); wassert(actual(CSVReader::unescape("\"\"\"\"\"")) == "\"\""); wassert(actual(CSVReader::unescape("a\"b")) == "a\"b"); }); // Test CSV reader add_method("reader", []() { { stringstream in(""); CSVReader reader(in); wassert(actual(reader.next()).isfalse()); } { stringstream in("\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 1); wassert(actual(reader.cols[0]) == ""); wassert(actual(reader.next()).isfalse()); } { stringstream in("\r\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 1); wassert(actual(reader.cols[0]) == ""); wassert(actual(reader.next()).isfalse()); } { stringstream in("1,2\r\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 2u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.cols[1]) == "2"); wassert(actual(reader.next()).isfalse()); } { stringstream in("1\r\n2\r\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 1u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 1u); wassert(actual(reader.cols[0]) == "2"); wassert(actual(reader.next()).isfalse()); } { stringstream in("1,2\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 2u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.cols[1]) == "2"); wassert(actual(reader.next()).isfalse()); } { stringstream in( "1,\",\",2\n" "antani,,blinda\n" ",\n" ); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 3u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.cols[1]) == ","); wassert(actual(reader.cols[2]) == "2"); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 3u); wassert(actual(reader.cols[0]) == "antani"); wassert(actual(reader.cols[1]) == ""); wassert(actual(reader.cols[2]) == "blinda"); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 2u); wassert(actual(reader.cols[0]) == ""); wassert(actual(reader.cols[1]) == ""); wassert(actual(reader.next()).isfalse()); } { stringstream in("1,2"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 2u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.cols[1]) == "2"); wassert(actual(reader.next()).isfalse()); } { stringstream in("\"1\"\r\n"); CSVReader reader(in); wassert(actual(reader.next()).istrue()); wassert(actual(reader.cols.size()) == 1u); wassert(actual(reader.cols[0]) == "1"); wassert(actual(reader.next()).isfalse()); } }); // Test write/read cycles add_method("writer", []() { MemoryCSVWriter out; out.add_value(1); out.add_value("\""); out.add_value("'"); out.add_value(","); out.add_value("\n"); out.flush_row(); out.buf.seekg(0); CSVReader in(out.buf); wassert(actual(in.next()).istrue()); wassert(actual(in.cols.size()) == 5); wassert(actual(in.as_int(0)) == 1); wassert(actual(in.cols[1]) == "\""); wassert(actual(in.cols[2]) == "'"); wassert(actual(in.cols[3]) == ","); wassert(actual(in.cols[4]) == "\n"); wassert(actual(in.next()).isfalse()); }); } } test("core_csv"); } dballe-8.6/dballe/core/json-test.cc0000644000175000017500000000550313554564112014154 00000000000000#include "tests.h" #include "json.h" #include using namespace std; using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("null", []() { // null value stringstream out; core::JSONWriter writer(out); writer.add_null(); wassert(actual(out.str()) == "null"); }); add_method("bool", []() { // bool value stringstream out; core::JSONWriter writer(out); writer.add(true); wassert(actual(out.str()) == "true"); out.str(""); writer.add(false); wassert(actual(out.str()) == "false"); }); add_method("int", []() { // int value stringstream out; core::JSONWriter writer(out); writer.add(1); wassert(actual(out.str()) == "1"); out.str(""); writer.add(-1234567); wassert(actual(out.str()) == "-1234567"); }); add_method("double", []() { // double value stringstream out; core::JSONWriter writer(out); writer.add(1.1); wassert(actual(out.str()) == "1.100000"); out.str(""); writer.add(-1.1); wassert(actual(out.str()) == "-1.100000"); out.str(""); writer.add(1.0); wassert(actual(out.str()) == "1.0"); out.str(""); writer.add(-1.0); wassert(actual(out.str()) == "-1.0"); }); add_method("string", []() { // string value stringstream out; core::JSONWriter writer(out); writer.add(""); wassert(actual(out.str()) == "\"\""); out.str(""); writer.add("antani"); wassert(actual(out.str()) == "\"antani\""); out.str(""); writer.add("\n"); wassert(actual(out.str()) == "\"\\n\""); }); add_method("list", []() { // list value stringstream out; core::JSONWriter writer(out); writer.start_list(); writer.add(""); writer.add(1); writer.add(1.0); writer.end_list(); wassert(actual(out.str()) == "[\"\",1,1.0]"); }); add_method("mapping", []() { // mapping value stringstream out; core::JSONWriter writer(out); writer.start_mapping(); writer.add("", 1); writer.add("antani", 1.0); writer.end_mapping(); wassert(actual(out.str()) == "{\"\":1,\"antani\":1.0}"); }); }; } test("core_json"); } dballe-8.6/dballe/core/string.h0000644000175000017500000000055713554564112013402 00000000000000#ifndef DBALLE_CORE_STRING_H #define DBALLE_CORE_STRING_H #include namespace dballe { /** * Look for the given argument in the URL query string, remove it from the url * string and return its value. * * If the argument is not found, returns false. */ bool url_pop_query_string(std::string& url, const std::string& name, std::string& val); } #endif dballe-8.6/dballe/core/var.h0000644000175000017500000000304713554564112012661 00000000000000#ifndef DBALLE_CORE_VAR_H #define DBALLE_CORE_VAR_H /** @file * Shortcut functions to work with wreport::Var in DB-All.e */ #include #include #include namespace dballe { /** * Convenience functions to quickly create variables from the local B table */ /// Resolve a comma-separated varcode list performing careful validation, inserting results in \a out void resolve_varlist(const std::string& varlist, std::set& out); /// Resolve a comma-separated varcode list performing careful validation, calling \a dest on each result void resolve_varlist(const std::string& varlist, std::function out); /// Create a new Var, copying \a var and all its attributes except the unset ones std::unique_ptr var_copy_without_unset_attrs(const wreport::Var& var); /** * Create a new Var with code \a code, copying the value from \a var and all * its attributes except the unset ones */ std::unique_ptr var_copy_without_unset_attrs(const wreport::Var& var, wreport::Varcode code); /** * Format the code to its string representation * * The string will be written to buf, which must be at least 7 bytes long */ void format_code(wreport::Varcode code, char* buf); /** * Format the B code to its string representation * * The string will be written to buf, which must be at least 7 bytes long */ void format_bcode(wreport::Varcode code, char* buf); /// Return \a code, or its DB-All.e equivalent wreport::Varcode map_code_to_dballe(wreport::Varcode code); } #endif dballe-8.6/dballe/core/structbuf-test.cc0000644000175000017500000000360713554564112015227 00000000000000#include "tests.h" #include "structbuf.h" using namespace dballe; using namespace wreport; using namespace dballe::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { // Test an in-memory structbuf add_method("memory", []() { Structbuf buf; wassert(actual(buf.size()) == 0); wassert(actual(buf.is_file_backed()).isfalse()); buf.append(1); wassert(actual(buf.size()) == 1); wassert(actual(buf.is_file_backed()).isfalse()); buf.append(2); wassert(actual(buf.size()) == 2); wassert(actual(buf.is_file_backed()).isfalse()); buf.append(3); wassert(actual(buf.size()) == 3); wassert(actual(buf.is_file_backed()).isfalse()); buf.ready_to_read(); for (unsigned i = 0; i < 3; ++i) wassert(actual(buf[i]) == i + 1); }); // Test an file-backed structbuf add_method("file", []() { Structbuf buf; buf.append(1); buf.append(2); buf.append(3); wassert(actual(buf.size()) == 3); wassert(actual(buf.is_file_backed()).isfalse()); buf.append(4); wassert(actual(buf.size()) == 4); wassert(actual(buf.is_file_backed()).istrue()); buf.append(5); wassert(actual(buf.size()) == 5); buf.append(6); wassert(actual(buf.size()) == 6); buf.append(7); wassert(actual(buf.size()) == 7); buf.append(8); wassert(actual(buf.size()) == 8); buf.ready_to_read(); for (unsigned i = 0; i < 8; ++i) wassert(actual(buf[i]) == i + 1); }); } } test("core_structbuf"); } dballe-8.6/dballe/core/aliases.gperf0000644000175000017500000000465113554573614014377 00000000000000/* * wreport/aliases - Aliases for commonly used variable codes * * Copyright (C) 2005--2011 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ %define slot-name alias %define class-name VarcodeAliases %define lookup-function-name find %struct-type %language=C++ %global-table %compare-strncmp /* Using %switch may be faster (remember to check when doing optimizations) */ %{ #include #include using namespace wreport; namespace dballe { %} struct aliasdef { const char* alias; Varcode var; }; %% block, WR_VAR(0, 1, 1) station, WR_VAR(0, 1, 2) height, WR_VAR(0, 7, 30) heightbaro, WR_VAR(0, 7, 31) name, WR_VAR(0, 1, 19) p, WR_VAR(0, 10, 4) mslp, WR_VAR(0, 10, 51) u, WR_VAR(0, 11, 3) v, WR_VAR(0, 11, 4) t, WR_VAR(0, 12, 101) td, WR_VAR(0, 12, 103) q, WR_VAR(0, 13, 1) rh, WR_VAR(0, 13, 3) tp, WR_VAR(0, 13, 11) mwd, WR_VAR(0, 22, 1) swh, WR_VAR(0, 22, 70) pp1d, WR_VAR(0, 22, 71) mwp, WR_VAR(0, 22, 74) conf, WR_VAR(0, 33, 7) data_id, WR_VAR(0, 33, 195) no, WR_VAR(0, 15, 192) no2, WR_VAR(0, 15, 193) o3, WR_VAR(0, 15, 194) pm10, WR_VAR(0, 15, 195) %% Varcode varcode_alias_resolve(const char* alias) { struct aliasdef* res = VarcodeAliases::find(alias, strlen(alias)); if (res == NULL) return 0; else return res->var; } Varcode varcode_alias_resolve_substring(const char* alias, int len) { struct aliasdef* res = VarcodeAliases::find(alias, len); if (res == NULL) return 0; else return res->var; } wreport::Varcode varcode_alias_resolve(const std::string& alias) { struct aliasdef* res = VarcodeAliases::find(alias.data(), alias.size()); if (res == NULL) return 0; else return res->var; } } /* vim:set ts=4 sw=4: */ dballe-8.6/dballe/core/varmatch-test.cc0000644000175000017500000001347513554564112015017 00000000000000#include "tests.h" #include "var.h" #include "varmatch.h" using namespace wreport; using namespace dballe; using namespace dballe::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("int", []() { Var var(varinfo(WR_VAR(0, 1, 1)), 42); wassert(actual((*Varmatch::parse("B01001<43"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001<42"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001<41"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001<=43"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001<=42"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001<=41"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001>41"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001>42"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001>43"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001>=41"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001>=42"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001>=43"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001==42"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001=42"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001==43"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001=43"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01001<>43"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01001<>42"))(var)).isfalse()); wassert(actual((*Varmatch::parse("41<=B01001<=42"))(var)).istrue()); wassert(actual((*Varmatch::parse("42<=B01001<=42"))(var)).istrue()); wassert(actual((*Varmatch::parse("42<=B01001<=43"))(var)).istrue()); wassert(actual((*Varmatch::parse("40<=B01001<=41"))(var)).isfalse()); }); add_method("decimal", []() { Var var(varinfo(WR_VAR(0, 12, 101)), 273.15); wassert(actual((*Varmatch::parse("B12101<274"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101<273.15"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101<273"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101<=274"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101<=273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101<=273"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101>273"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101>273.15"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101>274"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101>=273"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101>=273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101>=274"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101==273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101=273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101==274"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101=274"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B12101<>274"))(var)).istrue()); wassert(actual((*Varmatch::parse("B12101<>273.15"))(var)).isfalse()); wassert(actual((*Varmatch::parse("273<=B12101<=273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("273.15<=B12101<=273.15"))(var)).istrue()); wassert(actual((*Varmatch::parse("273.15<=B12101<=274"))(var)).istrue()); wassert(actual((*Varmatch::parse("272<=B12101<=273"))(var)).isfalse()); }); add_method("string", []() { Var var(varinfo(WR_VAR(0, 1, 11)), "enrico"); wassert(actual((*Varmatch::parse("B01011emanuele"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011>enrico"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01011>paolo"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01011>=emanuele"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011>=enrico"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011>=paolo"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01011==enrico"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011=enrico"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011==paolo"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01011=paolo"))(var)).isfalse()); wassert(actual((*Varmatch::parse("B01011<>paolo"))(var)).istrue()); wassert(actual((*Varmatch::parse("B01011<>enrico"))(var)).isfalse()); wassert(actual((*Varmatch::parse("emanuele<=B01011<=enrico"))(var)).istrue()); wassert(actual((*Varmatch::parse("enrico<=B01011<=enrico"))(var)).istrue()); wassert(actual((*Varmatch::parse("enrico<=B01011<=paolo"))(var)).istrue()); wassert(actual((*Varmatch::parse("daniele<=B01011<=emanuele"))(var)).isfalse()); }); } } test("core_varmatch"); } dballe-8.6/dballe/core/aliases.h0000644000175000017500000000203313554573614013513 00000000000000#ifndef WREPORT_CORE_ALIASES_H #define WREPORT_CORE_ALIASES_H /** @file * @ingroup core * Resolve aliases to variable codes */ #include #include namespace dballe { /** * Resolve a variable alias. * * @param alias * The alias to resolve * @return * The varcode corresponding to the alias, or 0 if no variable has the given * alias. */ wreport::Varcode varcode_alias_resolve(const char* alias); /** * Resolve a variable alias. * * @param alias * The alias to resolve (does not need to be null-terminated) * @param len * The length of the string * @return * The varcode corresponding to the aliase, or 0 if no variable has the given * alias. */ wreport::Varcode varcode_alias_resolve_substring(const char* alias, int len); /** * Resolve a variable alias. * * @param alias * The alias to resolve * @return * The varcode corresponding to the alias, or 0 if no variable has the given * alias. */ wreport::Varcode varcode_alias_resolve(const std::string& alias); } #endif dballe-8.6/dballe/core/vasprintf.h0000644000175000017500000000375313554564112014111 00000000000000#ifndef DBALLE_CORE_VASPRINTF_H #define DBALLE_CORE_VASPRINTF_H #include "config.h" #include #include #include #include #if USE_OWN_VASPRINTF static int vasprintf (char **result, const char *format, va_list args) { const char *p = format; /* Add one to make sure that it is never zero, which might cause malloc to return NULL. */ int total_width = strlen (format) + 1; va_list ap; memcpy ((void *)&ap, (void *)&args, sizeof (va_list)); while (*p != '\0') { if (*p++ == '%') { while (strchr ("-+ #0", *p)) ++p; if (*p == '*') { ++p; total_width += abs (va_arg (ap, int)); } else total_width += strtoul (p, (char **) &p, 10); if (*p == '.') { ++p; if (*p == '*') { ++p; total_width += abs (va_arg (ap, int)); } else total_width += strtoul (p, (char **) &p, 10); } while (strchr ("hlL", *p)) ++p; /* Should be big enough for any format specifier except %s and floats. */ total_width += 30; switch (*p) { case 'd': case 'i': case 'o': case 'u': case 'x': case 'X': case 'c': (void) va_arg (ap, int); break; case 'f': case 'e': case 'E': case 'g': case 'G': (void) va_arg (ap, double); /* Since an ieee double can have an exponent of 307, we'll make the buffer wide enough to cover the gross case. */ total_width += 307; break; case 's': total_width += strlen (va_arg (ap, char *)); break; case 'p': case 'n': (void) va_arg (ap, char *); break; } p++; } } *result = (char*)malloc (total_width); if (*result != NULL) { return vsprintf (*result, format, args);} else { return 0; } } static int asprintf (char **result, const char *format, ...) { va_list ap; va_start(ap, format); int res = vasprintf(result, format, ap); va_end(ap); return res; } #endif #endif dballe-8.6/dballe/core/data.cc0000644000175000017500000001422513554564112013140 00000000000000#include "data.h" #include #include using namespace wreport; namespace dballe { namespace core { Data::~Data() { } const Data& Data::downcast(const dballe::Data& data) { const Data* ptr = dynamic_cast(&data); if (!ptr) throw error_consistency("data given is not a core::Data"); return *ptr; } Data& Data::downcast(dballe::Data& data) { Data* ptr = dynamic_cast(&data); if (!ptr) throw error_consistency("data given is not a core::Data"); return *ptr; } void Data::validate() { datetime.set_lower_bound(); } #if 0 void Record::set_datetime(const Datetime& dt) { if (dt.is_missing()) { datetime = DatetimeRange(); } else { datetime.min = datetime.max = dt; } } void Record::set_datetimerange(const DatetimeRange& range) { datetime = range; } void Record::set_level(const Level& lev) { level = lev; } void Record::set_trange(const Trange& tr) { trange = tr; } void Record::set_var(const wreport::Var& var) { if (var.isset()) obtain(var.code()).setval(var); else unset_var(var.code()); } void Record::set_var_acquire(std::unique_ptr&& var) { int pos = find_item(var->code()); if (pos == -1) { // Insertion sort the new variable // Enlarge the buffer m_vars.resize(m_vars.size() + 1); /* Insertionsort. Crude, but our datasets should be too small for an * RB-Tree to be worth it */ for (pos = m_vars.size() - 1; pos > 0; --pos) if (m_vars[pos - 1]->code() > var->code()) m_vars[pos] = m_vars[pos - 1]; else break; } else delete m_vars[pos]; m_vars[pos] = var.release(); } void Record::set_latrange(const LatRange& lr) { latrange = lr; } void Record::set_lonrange(const LonRange& lr) { lonrange = lr; } #endif bool Data::operator==(const dballe::Data& other) const { const auto& o = downcast(other); return std::tie(station, datetime, level, trange, values) == std::tie(o.station, o.datetime, o.level, o.trange, o.values); } bool Data::operator!=(const dballe::Data& other) const { const auto& o = downcast(other); return std::tie(station, datetime, level, trange, values) != std::tie(o.station, o.datetime, o.level, o.trange, o.values); } void Data::clear_ids() { station.id = MISSING_INT; values.clear_ids(); } void Data::clear_vars() { values.clear(); } void Data::clear() { station = DBStation(); datetime = Datetime(); level = Level(); trange = Trange(); values.clear(); } #if 0 const std::vector& Record::vars() const { return m_vars; } Coords Record::get_coords() const { return station.coords; } Ident Record::get_ident() const { return station.ident; } Level Record::get_level() const { return level; } Trange Record::get_trange() const { return trange; } Datetime Record::get_datetime() const { if (datetime.min != datetime.max) return Datetime(); Datetime res = datetime.min; res.set_lower_bound(); return res; } DatetimeRange Record::get_datetimerange() const { DatetimeRange res = datetime; res.min.set_lower_bound(); res.max.set_upper_bound(); return res; } Station Record::get_station() const { return station; } DBStation Record::get_dbstation() const { return station; } const wreport::Var* Record::get_var(wreport::Varcode code) const { int pos = find_item(code); if (pos == -1) return NULL; return m_vars[pos]; } void Record::set_coords(const Coords& c) { station.coords = c; } void Record::set_station(const Station& s) { station.report = s.report; station.coords = s.coords; station.ident = s.ident; mobile = station.ident.is_missing() ? 0 : 1; } void Record::set_dbstation(const DBStation& s) { station = s; mobile = station.ident.is_missing() ? 0 : 1; } #endif void Data::set_from_string(const char* str) { // Split the input as name=val const char* s = strchr(str, '='); if (!s) error_consistency::throwf("there should be an = between the name and the value in '%s'", str); std::string key(str, s - str); setf(key.data(), key.size(), s + 1); } void Data::set_from_test_string(const std::string& s) { if (s.empty()) return; size_t cur = 0; while (true) { size_t next = s.find(", ", cur); if (next == std::string::npos) { set_from_string(s.substr(cur).c_str()); break; } else { set_from_string(s.substr(cur, next - cur).c_str()); cur = next + 2; } } validate(); } namespace { struct BufferPrinter { std::stringstream s; bool first = true; template void print(const KEY& key, const VAL& val) { if (first) first = false; else s << ","; s << key << "=" << val; } }; struct FilePrinter { FILE* out; template void print(const char* key, const VAL& val) { fprintf(out, "%s=", key); val.print(out); } void print(const char* key, int val) { fprintf(out, "%s=%d\n", key, val); } void print(const char* key, const std::string& val) { fprintf(out, "%s=%s\n", key, val.c_str()); } }; } std::string Data::to_string() const { BufferPrinter printer; if (!station.is_missing()) printer.print("station", station); if (!datetime.is_missing()) printer.print("datetime", datetime); if (!level.is_missing()) printer.print("level", level); if (!trange.is_missing()) printer.print("trange", trange); for (const auto& val: values) printer.print(varcode_format(val.code()), val); return printer.s.str(); } void Data::print(FILE* out) const { FilePrinter printer; printer.out = out; if (!station.is_missing()) printer.print("station", station); if (!datetime.is_missing()) printer.print("datetime", datetime); if (!level.is_missing()) printer.print("level", level); if (!trange.is_missing()) printer.print("trange", trange); for (const auto& var: values) var.print(out); } } } dballe-8.6/dballe/core/defs.cc0000644000175000017500000000036613554564112013151 00000000000000#include "defs.h" #include #include #include #include #include #include #include #include using namespace std; using namespace wreport; namespace dballe { } dballe-8.6/dballe/core/data.h0000644000175000017500000000735013554564112013003 00000000000000#ifndef DBALLE_CORE_DATA_H #define DBALLE_CORE_DATA_H #include #include namespace dballe { namespace core { /** * Holds data for database I/O * * Data is a container for one observation of meteorological values, that * includes station informations, physical location of the observation in time * and space, and all the observed variables. */ class Data : public dballe::Data { public: DBStation station; Datetime datetime; Level level; Trange trange; DBValues values; public: Data() = default; Data(const Data& rec) = default; Data(Data&& rec) = default; ~Data(); Data& operator=(const Data& rec) = default; Data& operator=(Data&& rec) = default; bool operator==(const dballe::Data& rec) const override; bool operator!=(const dballe::Data& rec) const override; /** * Check the data fields for consistency, and fill in missing values: * * - month without year, day without month, and so on, cause errors * - min and max datetimes are filled with the actual minimum and maximum * values acceptable for that range (year=2017, for example, becomes * min=2017-01-01 00:00:00, max=2017-12-31 23:59:59 */ void validate(); void clear() override; void clear_vars() override; void clear_ids() override; #if 0 void set_datetime(const Datetime& dt) override; void set_datetimerange(const DatetimeRange& range) override; void set_coords(const Coords& c) override; void set_latrange(const LatRange& lr) override; void set_lonrange(const LonRange& lr) override; void set_level(const Level& lev) override; void set_trange(const Trange& tr) override; void set_var(const wreport::Var& var) override; void set_var_acquire(std::unique_ptr&& var) override; void set_station(const Station& s) override; void set_dbstation(const DBStation& s) override; #endif void print(FILE* out) const override; /** * Return a reference to record downcasted as core::Record. * * Throws an exception if record is not a core::Record. */ static const Data& downcast(const dballe::Data& data); /** * Return a reference to record downcasted as core::Record. * * Throws an exception if record is not a core::Record. */ static Data& downcast(dballe::Data& data); #if 0 Coords get_coords() const override; Ident get_ident() const override; Level get_level() const override; Trange get_trange() const override; Datetime get_datetime() const override; DatetimeRange get_datetimerange() const override; Station get_station() const override; DBStation get_dbstation() const override; const wreport::Var* get_var(wreport::Varcode code) const override; #endif /** * Set a value according to an assignment encoded in a string. * * String can use keywords, aliases and varcodes. Examples: ana_id=3, * name=Bologna, B12012=32.4 * * In case of numeric parameter, a hyphen ("-") means MISSING_INT (e.g., * `leveltype2=-`). * * @param str * The string containing the assignment. */ void set_from_string(const char* str); /** * Set the Data from a ", "-separated string of assignments. * * The implementation is not efficient and the method is not safe for any * input, since ", " could appear in a station identifier. It is however * useful to quickly create test queries for unit testing. */ void set_from_test_string(const std::string& s); /** * Encode in a one-liner of comma-separated assignments */ std::string to_string() const; protected: void setf(const char* key, unsigned len, const char* val); }; } } #endif dballe-8.6/dballe/core/enq.h0000644000175000017500000001427213554564112012656 00000000000000#ifndef DBALLE_CORE_ENQ_H #define DBALLE_CORE_ENQ_H #include #include #include #include #include #include namespace dballe { namespace impl { /** * Class passed to key-value accessors to set values in an invoker-defined way */ struct Enq { const char* key; unsigned len; bool missing = true; Enq(const char* key, unsigned len) : key(key), len(len) { } virtual ~Enq() {} [[noreturn]] void throw_notfound() { wreport::error_notfound::throwf("key %s not found on this query result", key); } // Return the name of this access operation virtual const char* name() const = 0; // Set a boolean value virtual void set_bool(bool val) = 0; // Set an always defined int value virtual void set_int(int val) = 0; // Set an int value that is undefined if it is MISSING_INT virtual void set_dballe_int(int val) = 0; // Set a string virtual void set_string(const std::string& val) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set station identifier virtual void set_ident(const Ident& ident) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set variable code virtual void set_varcode(wreport::Varcode val) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set variable value virtual void set_var(const wreport::Var* val) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } /** * Set the value using the value of the given variable. * * var can be assumed to always be set, that is, the caller has already * checked that it has a value. */ virtual void set_var_value(const wreport::Var& var) = 0; // Set variable attributes virtual void set_attrs(const wreport::Var* val) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set latitude virtual void set_lat(int lat) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set longitude virtual void set_lon(int lon) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set coordinates virtual void set_coords(const Coords& c) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set station virtual void set_station(const Station& s) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set station with DB info virtual void set_dbstation(const DBStation& s) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set datetime virtual void set_datetime(const Datetime& dt) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set level virtual void set_level(const Level& dt) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } // Set timerange virtual void set_trange(const Trange& dt) { wreport::error_consistency::throwf("cannot %s `%s`", name(), key); } template bool search_b_values(const Values& values) { if (key[0] != 'B' || len != 6) return false; wreport::Varcode code = WR_STRING_TO_VAR(key + 1); const wreport::Var* var = values.maybe_var(code); if (var && var->isset()) set_var_value(*var); return true; } bool search_b_value(const dballe::Value& value) { if (key[0] != 'B' || len != 6) return false; wreport::Varcode code = WR_STRING_TO_VAR(key + 1); if (code != value.code()) throw_notfound(); const wreport::Var* var = value.get(); if (var && var->isset()) set_var_value(*var); return true; } template void search_alias_values(const Values& values) { wreport::Varcode code = dballe::resolve_varcode(key); const wreport::Var* var = values.maybe_var(code); if (var && var->isset()) set_var_value(*var); } void search_alias_value(const dballe::Value& value) { wreport::Varcode code = dballe::resolve_varcode(key); if (code != value.code()) throw_notfound(); const wreport::Var* var = value.get(); if (var && var->isset()) set_var_value(*var); } }; struct Enqi : public Enq { using Enq::Enq; int res; const char* name() const override { return "enqi"; } void set_bool(bool val) override { res = val ? 1 : 0; missing = false; } void set_int(int val) override { res = val; missing = false; } void set_dballe_int(int val) override { if (val == MISSING_INT) return; res = val; missing = false; } void set_lat(int lat) override { if (lat == MISSING_INT) return; res = lat; missing = false; } void set_lon(int lon) override { if (lon == MISSING_INT) return; res = lon; missing = false; } void set_var_value(const wreport::Var& var) override { missing = false; res = var.enqi(); } }; struct Enqd : public Enq { using Enq::Enq; double res; const char* name() const override { return "enqd"; } void set_bool(bool val) override { res = val ? 1 : 0; missing = false; } void set_int(int val) override { res = val; missing = false; } void set_dballe_int(int val) override { if (val == MISSING_INT) return; res = val; missing = false; } void set_lat(int lat) override { if (lat == MISSING_INT) return; res = Coords::lat_from_int(lat); missing = false; } void set_lon(int lon) override { if (lon == MISSING_INT) return; res = Coords::lon_from_int(lon); missing = false; } void set_var_value(const wreport::Var& var) override { missing = false; res = var.enqd(); } }; } } #endif dballe-8.6/dballe/core/match-wreport-test.cc0000644000175000017500000005027113554564112016001 00000000000000#include "tests.h" #include "match-wreport.h" #include "var.h" #include #include #include #include #include #include #include using namespace std; using namespace dballe; using namespace dballe::tests; using namespace wreport; namespace { std::unique_ptr get_matcher(const char* q) { return Matcher::create(*dballe::tests::query_from_string(q)); } struct Fixture : public dballe::tests::Fixture { Tables tables; Fixture() { tables.load_bufr(BufrTableID(0, 0, 0, 24, 0)); } }; class TestSubset : public FixtureTestCase { using FixtureTestCase::FixtureTestCase; void register_tests() override { // Test station_id matcher add_method("station_id", [](Fixture& f) { auto m = get_matcher("ana_id=1"); Tables tables; tables.load_bufr(BufrTableID(200, 0, 0, 14, 1)); Subset s(tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 1, 192), 1); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); }); // Test station WMO matcher add_method("station_wmo", [](Fixture& f) { { auto m = get_matcher("block=11"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.back().seti(11); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s.store_variable_i(WR_VAR(0, 1, 2), 222); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("block=11, station=222"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.back().seti(11); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 1, 2), 22); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.back().seti(222); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].seti(1); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0] = var(WR_VAR(0, 1, 192)); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); } }); // Test date matcher add_method("date", [](Fixture& f) { { auto m = get_matcher("yearmin=2000"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 4, 1), 1999); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmax=2000"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 4, 1), 2001); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmin=2000, yearmax=2010"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_i(WR_VAR(0, 4, 1), 1999); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].seti(2011); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].seti(2005); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].seti(2010); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } }); // Test coordinates matcher add_method("coords", [](Fixture& f) { { auto m = get_matcher("latmin=45.00"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 5, 1), 43.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].setd(45.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].setd(46.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("latmax=45.00"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 5, 1), 46.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].setd(45.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].setd(44.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=11.00, lonmax=180.0"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 6, 1), 10.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].setd(11.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].setd(12.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=-180, lonmax=11.0"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 6, 1), 12.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].setd(11.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); s[0].setd(10.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } { auto m = get_matcher("latmin=45.0, latmax=46.0, lonmin=10.0, lonmax=12.0"); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 5, 1), 45.5); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s.store_variable_d(WR_VAR(0, 6, 1), 13.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[1].setd(11.0); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); } }); // Test rep_memo matcher add_method("rep_memo", [](Fixture& f) { auto m = get_matcher("rep_memo=synop"); Tables tables; tables.load_bufr(BufrTableID(200, 0, 0, 14, 1)); Subset s(tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); wassert(s.store_variable_c(WR_VAR(0, 1, 194), "temp")); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_NO); s[0].setc("synop"); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); }); // Test empty matcher add_method("empty", [](Fixture& f) { std::unique_ptr m = Matcher::create(*Query::create()); Subset s(f.tables); wassert(actual_matcher_result(m->match(MatchedSubset(s))) == matcher::MATCH_YES); }); } } test1("core_match_wreport_subset"); struct FixtureBulletin : public dballe::tests::Fixture { BufrBulletin* bulletin = nullptr; ~FixtureBulletin() { delete bulletin; } BufrBulletin& get_bulletin() { delete bulletin; bulletin = BufrBulletin::create().release(); BufrBulletin& b = *bulletin; b.edition_number = 4; b.originating_centre = 200; b.originating_subcentre = 0; b.master_table_version_number = 14; b.master_table_version_number_local = 0; b.load_tables(); return b; } }; class TestBulletin : public FixtureTestCase { using FixtureTestCase::FixtureTestCase; void register_tests() override { // Test station_id matcher add_method("station_id", [](Fixture& f) { auto m = get_matcher("ana_id=1"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1).store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 1, 192), 1); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); }); // Test station WMO matcher add_method("station_wmo", [](Fixture& f) { { auto m = get_matcher("block=11"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1).store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1).back().seti(11); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(1).store_variable_i(WR_VAR(0, 1, 2), 222); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("block=11, station=222"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 1, 1), 1); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).back().seti(11); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 1, 2), 22); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).back().seti(222); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(0)[0].seti(1); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0] = var(WR_VAR(0, 1, 192)); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); } // Valid block and station must be in the same subset { auto m = get_matcher("block=11, station=222"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 1, 1), 11); b.obtain_subset(1).store_variable_i(WR_VAR(0, 1, 2), 222); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 1, 2), 222); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } }); // Test date matcher add_method("date", [](Fixture& f) { { auto m = get_matcher("yearmin=2000"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 4, 1), 1999); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmax=2000"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 4, 1), 2001); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("yearmin=2000, yearmax=2010"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_i(WR_VAR(0, 4, 1), 1999); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].seti(2011); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].seti(2000); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(0)[0].seti(2005); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(0)[0].seti(2010); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } }); // Test coordinates matcher add_method("coords", [](Fixture& f) { { auto m = get_matcher("latmin=45.0"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1).store_variable_d(WR_VAR(0, 5, 1), 43.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1)[0].setd(45.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(1)[0].setd(46.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("latmax=45.0"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1).store_variable_d(WR_VAR(0, 5, 1), 46.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(1)[0].setd(45.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(1)[0].setd(44.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=11.00, lonmax=180.0"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_d(WR_VAR(0, 6, 1), 10.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].setd(11.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(0)[0].setd(12.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("lonmin=-180, lonmax=11.0"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_d(WR_VAR(0, 6, 1), 12.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].setd(11.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); b.obtain_subset(0)[0].setd(10.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } { auto m = get_matcher("latmin=45.0, latmax=46.0, lonmin=10.0, lonmax=12.0"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_d(WR_VAR(0, 5, 1), 45.5); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_d(WR_VAR(0, 6, 1), 13.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[1].setd(11.0); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); } }); // Test rep_memo matcher add_method("rep_memo", [](Fixture& f) { auto m = get_matcher("rep_memo=synop"); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0).store_variable_c(WR_VAR(0, 1, 194), "temp"); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_NO); b.obtain_subset(0)[0].setc("synop"); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); }); // Test empty matcher add_method("empty", [](Fixture& f) { std::unique_ptr m = Matcher::create(*Query::create()); BufrBulletin& b = f.get_bulletin(); wassert(actual_matcher_result(m->match(MatchedBulletin(b))) == matcher::MATCH_YES); }); } } test2("core_match_wreport_bulletin"); } dballe-8.6/dballe/core/query-test.cc0000644000175000017500000001744613572423710014357 00000000000000#include "dballe/core/tests.h" #include "dballe/core/query.h" #include using namespace std; using namespace dballe; using namespace dballe::tests; using namespace wreport; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_query"); void Tests::register_tests() { add_method("all_unset", []() { core::Query q; wassert(actual(q.ana_id) == MISSING_INT); wassert(actual(q.priomin) == MISSING_INT); wassert(actual(q.priomax) == MISSING_INT); wassert(actual(q.report) == ""); wassert(actual(q.mobile) == MISSING_INT); wassert(actual(q.ident.is_missing()).istrue()); wassert(actual(q.latrange) == LatRange()); wassert(actual(q.lonrange) == LonRange()); wassert(actual(q.dtrange) == DatetimeRange()); wassert(actual(q.level) == Level()); wassert(actual(q.trange) == Trange()); wassert(actual(q.varcodes.size()) == 0); wassert(actual(q.query) == ""); wassert(actual(q.ana_filter) == ""); wassert(actual(q.data_filter) == ""); wassert(actual(q.attr_filter) == ""); wassert(actual(q.limit) == MISSING_INT); wassert(actual(q.block) == MISSING_INT); wassert(actual(q.station) == MISSING_INT); }); #if 0 add_method("all_set", []() { core::Query q; q.setf("ana_id", "4"); q.setf("priority", "1"); q.setf("rep_memo", "foo"); q.setf("mobile", "0"); q.setf("ident", "bar"); q.setf("lat", "44.123"); q.setf("lon", "11.123"); q.datetime.min = q.datetime.max = Datetime(2000, 1, 2, 12, 30, 45); q.level = Level(10, 11, 12, 13); q.trange = Trange(20, 21, 22); q.setf("var", "B12101"); q.setf("query", "best"); q.setf("ana_filter", "B01001=1"); q.setf("data_filter", "B12101>260"); q.setf("attr_filter", "B33007>50"); q.setf("limit", "100"); q.setf("block", "16"); q.setf("station", "404"); wassert(actual(q.ana_id) == 4); wassert(actual(q.prio_min) == 1); wassert(actual(q.prio_max) == 1); wassert(actual(q.rep_memo) == "foo"); wassert(actual(q.mobile) == 0); wassert(actual(q.ident.is_missing()).isfalse()); wassert(actual(q.ident.get()) == "bar"); wassert(actual(q.latrange) == LatRange(44.123, 44.123)); wassert(actual(q.lonrange) == LonRange(11.123, 11.123)); wassert(actual(q.datetime) == DatetimeRange(2000, 1, 2, 12, 30, 45, 2000, 1, 2, 12, 30, 45)); wassert(actual(q.level) == Level(10, 11, 12, 13)); wassert(actual(q.trange) == Trange(20, 21, 22)); wassert(actual(q.varcodes.size()) == 1); wassert(actual(*q.varcodes.begin()) == WR_VAR(0, 12, 101)); wassert(actual(q.query) == "best"); wassert(actual(q.ana_filter) == "B01001=1"); wassert(actual(q.data_filter) == "B12101>260"); wassert(actual(q.attr_filter) == "B33007>50"); wassert(actual(q.limit) == 100); wassert(actual(q.block) == 16); wassert(actual(q.station) == 404); }); #endif add_method("prio", []() { core::Query q; q.set_from_test_string("priority=11"); wassert(actual(q.priomin) == 11); wassert(actual(q.priomax) == 11); q.clear(); q.set_from_test_string("priomin=12"); wassert(actual(q.priomin) == 12); wassert(actual(q.priomax) == MISSING_INT); q.clear(); q.set_from_test_string("priomax=12"); wassert(actual(q.priomin) == MISSING_INT); wassert(actual(q.priomax) == 12); q.clear(); q.set_from_test_string("priomin=11, priomax=22"); wassert(actual(q.priomin) == 11); wassert(actual(q.priomax) == 22); q.clear(); q.set_from_test_string("priomin=11, priomax=22, priority=16"); wassert(actual(q.priomin) == 16); wassert(actual(q.priomax) == 16); }); add_method("lat", []() { core::Query q; q.set_from_test_string("lat=45.0"); wassert(actual(q.latrange) == LatRange(45.0, 45.0)); q.clear(); q.set_from_test_string("latmin=45.0"); wassert(actual(q.latrange) == LatRange(45.0, LatRange::DMAX)); q.clear(); q.set_from_test_string("latmax=45.0"); wassert(actual(q.latrange) == LatRange(LatRange::DMIN, 45.0)); q.clear(); q.set_from_test_string("latmin=40.0, latmax=50.0"); wassert(actual(q.latrange) == LatRange(40.0, 50.0)); q.clear(); q.set_from_test_string("latmin=40.0, latmax=50.0, lat=42.0"); wassert(actual(q.latrange) == LatRange(42.0, 42.0)); }); add_method("lon", []() { core::Query q; q.set_from_test_string("lon=45.0"); wassert(actual(q.lonrange) == LonRange(45.0, 45.0)); q.clear(); try { q.set_from_test_string("lonmin=45.0"); wassert(actual(false).istrue()); } catch (std::exception& e) { wassert(actual(e.what()).contains("open ended range")); } wassert(actual(q.lonrange) == LonRange()); q.clear(); q.set_from_test_string("lonmin=40.0, lonmax=50.0"); wassert(actual(q.lonrange) == LonRange(40.0, 50.0)); q.clear(); q.set_from_test_string("lonmin=40.0, lonmax=50.0, lon=42.0"); wassert(actual(q.lonrange) == LonRange(42.0, 42.0)); }); add_method("lonrange", []() { // See issue #132: setting lonmin/lonmax now normalizes at each set core::Query q; q.set_from_test_string("lonmin=0, lonmax=360.0"); wassert(actual(q.lonrange) == LonRange(0.0, 0.0)); }); add_method("dtrange", []() { core::Query q; wassert(q.set_from_test_string("year=2015")); wassert(actual(q.dtrange) == DatetimeRange(2015, 1, 1, 0 , 0, 0, 2015, 12, 31, 23, 59, 59)); q.clear(); wassert(q.set_from_test_string("year=2015, monthmin=1, monthmax=2")); wassert(actual(q.dtrange) == DatetimeRange(2015, 1, 1, 0 , 0, 0, 2015, 2, 28, 23, 59, 59)); q.clear(); auto e = wassert_throws(wreport::error_consistency, q.set_from_test_string("year=2015, monthmin=2, day=28")); wassert(actual(e.what()) == "day 28 given with no month"); q.clear(); e = wassert_throws(wreport::error_consistency, q.set_from_test_string("yearmin=2000, yearmax=2012, month=2, min=30")); wassert(actual(e.what()) == "minute 30 given with no hour"); q.clear(); wassert(q.set_from_test_string("yearmin=2010, yearmax=2012, year=2000, month=2, hour=12, min=30")); wassert(actual(q.dtrange) == DatetimeRange(2000, 2, 1, 12, 30, 0, 2000, 2, 29, 12, 30, 59)); q.clear(); e = wassert_throws(wreport::error_consistency, q.set_from_test_string("yearmin=2010, yearmax=2012, year=2000, month=2, min=30")); wassert(actual(e.what()) == "minute 30 given with no hour"); }); add_method("varlist", []() { core::Query q; q.set_from_test_string("var=B12101"); wassert(actual(q.varcodes.size()) == 1); wassert(actual(*q.varcodes.begin()) == WR_VAR(0, 12, 101)); q.clear(); q.set_from_test_string("varlist=B12101,B01001"); wassert(actual(q.varcodes.size()) == 2); wassert(actual(*q.varcodes.begin()) == WR_VAR(0, 1, 1)); wassert(actual(*q.varcodes.rbegin()) == WR_VAR(0, 12, 101)); q.clear(); q.set_from_test_string("varlist=B12101,B01001, var=B12102"); wassert(actual(q.varcodes.size()) == 1); wassert(actual(*q.varcodes.begin()) == WR_VAR(0, 12, 102)); }); add_method("modifiers", []() { wassert(actual(core::Query::parse_modifiers("best")) == DBA_DB_MODIFIER_BEST); wassert(actual(core::Query::parse_modifiers("details")) == DBA_DB_MODIFIER_SUMMARY_DETAILS); wassert(actual(core::Query::parse_modifiers("attrs")) == DBA_DB_MODIFIER_WITH_ATTRIBUTES); wassert(actual(core::Query::parse_modifiers("best,attrs")) == (DBA_DB_MODIFIER_BEST | DBA_DB_MODIFIER_WITH_ATTRIBUTES)); }); add_method("issue107", []() { core::Query q; wassert_throws(wreport::error_consistency, q.set_from_test_string("month=6")); }); add_method("setf_unset", []() { core::Query q; q.set_from_string("leveltype2=42"); wassert(actual(q.level.ltype2) == 42); q.set_from_string("leveltype2=-"); wassert(actual(q.level.ltype2) == MISSING_INT); }); } } dballe-8.6/dballe/core/arrayfile.h0000644000175000017500000000116013554564112014041 00000000000000#ifndef DBALLE_CORE_ARRAYFILE_H #define DBALLE_CORE_ARRAYFILE_H /** @file * @ingroup core * In-memory versions of File, to be used for testing, */ #include #include namespace dballe { namespace core { class ArrayFile : public dballe::core::File { protected: Encoding file_type; public: std::vector msgs; /// Current reading offset in msgs unsigned current; ArrayFile(Encoding type); virtual ~ArrayFile(); Encoding encoding() const override; BinaryMessage read() override; void write(const std::string& msg) override; }; } } #endif dballe-8.6/dballe/core/smallset-test.cc0000644000175000017500000000334513554564112015031 00000000000000#include "dballe/core/tests.h" #include "dballe/core/smallset.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_smallset"); struct IntSmallSet : public core::SmallSet { }; void Tests::register_tests() { add_method("by_1", []{ IntSmallSet s; for (int i = 0; i < 32; ++i) { s.add(i); auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } for (int i = 0; i < 32; ++i) { auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } wassert_true(s.find(33) == s.end()); }); add_method("by_5", []{ IntSmallSet s; for (int i = 0; i < 32; ++i) { for (int pad = 0; pad < 5; ++pad) s.add((i + 1) * 100 + pad); s.add(i); auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } for (int i = 0; i < 32; ++i) { auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } wassert_true(s.find(33) == s.end()); }); add_method("by_64", []{ IntSmallSet s; for (int i = 0; i < 32; ++i) { for (int pad = 0; pad < 64; ++pad) s.add((i + 1) * 100 + pad); s.add(i); auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } for (int i = 0; i < 32; ++i) { auto it = s.find(i); wassert_true(it != s.end()); wassert(actual(*it) == i); } wassert_true(s.find(33) == s.end()); }); } } dballe-8.6/dballe/core/fwd.h0000644000175000017500000000035213554564112012645 00000000000000#ifndef DBALLE_CORE_FWD_H #define DBALLE_CORE_FWD_H namespace dballe { namespace core { class Data; class Query; class JSONWriter; class JSONReader; namespace json { class Stream; } } namespace impl { struct Shortcut; } } #endif dballe-8.6/dballe/core/shortcuts-access.cc0000644000175000017500000011231413554564124015525 00000000000000#include "shortcuts.h" #include #include namespace dballe { namespace impl { const Shortcut& Shortcut::by_name(const char* key, unsigned len) { switch (len) { case 3: switch (key[0]) { case 'd': if (memcmp(key + 1, "ay", 2) == 0) { return sc::day; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'q': if (memcmp(key + 1, "nh", 2) == 0) { return sc::qnh; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { return sc::year; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { return sc::hour; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 5: switch (key[0]) { case 'b': if (memcmp(key + 1, "lock", 4) == 0) { return sc::block; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'i': if (memcmp(key + 1, "dent", 4) == 0) { return sc::ident; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { return sc::month; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'p': if (memcmp(key + 1, "ress", 4) == 0) { return sc::press; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 6: switch (key[0]) { case 'n': if (memcmp(key + 1, "avsys", 5) == 0) { return sc::navsys; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'm': if (memcmp(key + 1, "inute", 5) == 0) { return sc::minute; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': switch (key[1]) { case 'e': if (memcmp(key + 2, "cond", 4) == 0) { return sc::second; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': if (memcmp(key + 2, "_dir", 4) == 0) { return sc::st_dir; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 7: switch (key[0]) { case 's': if (key[1] == 't') { switch (key[2]) { case '_': switch (key[3]) { case 't': if (memcmp(key + 4, "ype", 3) == 0) { return sc::st_type; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'n': if (memcmp(key + 4, "ame", 3) == 0) { return sc::st_name; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'a': if (memcmp(key + 3, "tion", 4) == 0) { return sc::station; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': switch (key[1]) { case 'i': if (memcmp(key + 2, "mesig", 5) == 0) { return sc::timesig; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'e': if (memcmp(key + 2, "mp_2m", 5) == 0) { return sc::temp_2m; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'c': if (memcmp(key + 1, "loud_n", 6) == 0) { return sc::cloud_n; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 8: switch (key[0]) { case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { return sc::rep_memo; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'l': if (memcmp(key + 1, "atitude", 7) == 0) { return sc::latitude; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': if (memcmp(key + 1, "t_speed", 7) == 0) { return sc::st_speed; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'p': if (memcmp(key + 1, "res", 3) == 0) { switch (key[4]) { case 's': if (memcmp(key + 5, "_3h", 3) == 0) { return sc::press_3h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case '_': if (memcmp(key + 5, "wtr", 3) == 0) { return sc::pres_wtr; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': if (memcmp(key + 1, "ot_snow", 7) == 0) { return sc::tot_snow; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'h': if (memcmp(key + 1, "umidity", 7) == 0) { return sc::humidity; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'w': if (memcmp(key + 1, "ind_dir", 7) == 0) { return sc::wind_dir; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'c': if (memcmp(key + 1, "loud_", 5) == 0) { switch (key[6]) { case 'n': switch (key[7]) { case 'h': return sc::cloud_nh; break; case '1': return sc::cloud_n1; break; case '2': return sc::cloud_n2; break; case '3': return sc::cloud_n3; break; case '4': return sc::cloud_n4; break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'h': switch (key[7]) { case 'h': return sc::cloud_hh; break; case '1': return sc::cloud_h1; break; case '2': return sc::cloud_h2; break; case '3': return sc::cloud_h3; break; case '4': return sc::cloud_h4; break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'c': switch (key[7]) { case 'l': return sc::cloud_cl; break; case 'm': return sc::cloud_cm; break; case 'h': return sc::cloud_ch; break; case '1': return sc::cloud_c1; break; case '2': return sc::cloud_c2; break; case '3': return sc::cloud_c3; break; case '4': return sc::cloud_c4; break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 9: switch (key[0]) { case 'w': if (memcmp(key + 1, "ind_inst", 8) == 0) { return sc::wind_inst; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'l': if (memcmp(key + 1, "ongitude", 8) == 0) { return sc::longitude; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'p': if (memcmp(key + 1, "ress_", 5) == 0) { switch (key[6]) { case '2': if (memcmp(key + 7, "4h", 2) == 0) { return sc::press_24h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'm': if (memcmp(key + 7, "sl", 2) == 0) { return sc::press_msl; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'm': if (memcmp(key + 1, "etar_wtr", 8) == 0) { return sc::metar_wtr; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': if (memcmp(key + 1, "ot_prec", 7) == 0) { switch (key[8]) { case '1': return sc::tot_prec1; break; case '3': return sc::tot_prec3; break; case '6': return sc::tot_prec6; break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 10: switch (key[0]) { case 'p': switch (key[1]) { case 'o': if (memcmp(key + 2, "ll_", 3) == 0) { switch (key[5]) { case 'l': if (memcmp(key + 6, "code", 4) == 0) { return sc::poll_lcode; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': if (memcmp(key + 6, "code", 4) == 0) { return sc::poll_scode; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'a': if (memcmp(key + 6, "type", 4) == 0) { return sc::poll_atype; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': if (memcmp(key + 6, "type", 4) == 0) { return sc::poll_ttype; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'r': if (memcmp(key + 2, "ess_tend", 8) == 0) { return sc::press_tend; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'd': if (memcmp(key + 1, "ata_relay", 9) == 0) { return sc::data_relay; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': if (memcmp(key + 1, "onde_type", 9) == 0) { return sc::sonde_type; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'w': switch (key[1]) { case 'a': if (memcmp(key + 2, "ter_temp", 8) == 0) { return sc::water_temp; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'i': if (memcmp(key + 2, "nd_speed", 8) == 0) { return sc::wind_speed; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'v': if (memcmp(key + 1, "isibility", 9) == 0) { return sc::visibility; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 't': if (memcmp(key + 1, "ot_prec", 7) == 0) { switch (key[8]) { case '1': if (key[9] == '2') { return sc::tot_prec12; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case '2': if (key[9] == '4') { return sc::tot_prec24; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'e': if (memcmp(key + 1, "x_cw_wind", 9) == 0) { return sc::ex_cw_wind; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 11: switch (key[0]) { case 'p': if (memcmp(key + 1, "oll_source", 10) == 0) { return sc::poll_source; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'f': if (memcmp(key + 1, "light_roll", 10) == 0) { return sc::flight_roll; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'l': if (memcmp(key + 1, "atlon_spec", 10) == 0) { return sc::latlon_spec; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'h': if (memcmp(key + 1, "eight_", 6) == 0) { switch (key[7]) { case 'b': if (memcmp(key + 8, "aro", 3) == 0) { return sc::height_baro; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'a': if (memcmp(key + 8, "nem", 3) == 0) { return sc::height_anem; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'w': if (memcmp(key + 1, "et_temp_2m", 10) == 0) { return sc::wet_temp_2m; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'd': if (memcmp(key + 1, "ewpoint_2m", 10) == 0) { return sc::dewpoint_2m; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'e': if (memcmp(key + 1, "x_ccw_wind", 10) == 0) { return sc::ex_ccw_wind; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 12: switch (key[0]) { case 's': switch (key[1]) { case 't': switch (key[2]) { case '_': if (memcmp(key + 3, "name_icao", 9) == 0) { return sc::st_name_icao; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'a': if (memcmp(key + 3, "te_ground", 9) == 0) { return sc::state_ground; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'o': if (memcmp(key + 2, "nde_method", 10) == 0) { return sc::sonde_method; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'f': if (memcmp(key + 1, "light_phase", 11) == 0) { return sc::flight_phase; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'p': if (memcmp(key + 1, "ast_wtr", 7) == 0) { switch (key[8]) { case '1': if (key[9] == '_') { switch (key[10]) { case '3': if (key[11] == 'h') { return sc::past_wtr1_3h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case '6': if (key[11] == 'h') { return sc::past_wtr1_6h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case '2': if (key[9] == '_') { switch (key[10]) { case '3': if (key[11] == 'h') { return sc::past_wtr2_3h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case '6': if (key[11] == 'h') { return sc::past_wtr2_6h; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 13: switch (key[0]) { case 'p': if (memcmp(key + 1, "oll_gemscode", 12) == 0) { return sc::poll_gemscode; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'f': if (memcmp(key + 1, "light_reg_no", 12) == 0) { return sc::flight_reg_no; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 14: switch (key[0]) { case 't': if (memcmp(key + 1, "emp_precision", 13) == 0) { return sc::temp_precision; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'h': if (memcmp(key + 1, "eight_", 6) == 0) { switch (key[7]) { case 's': if (memcmp(key + 8, "tation", 6) == 0) { return sc::height_station; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 'r': if (memcmp(key + 8, "elease", 6) == 0) { return sc::height_release; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': if (memcmp(key + 1, "onde_tracking", 13) == 0) { return sc::sonde_tracking; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 15: if (memcmp(key + 0, "meas_equip_type", 15) == 0) { return sc::meas_equip_type; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 16: switch (key[0]) { case 'i': if (memcmp(key + 1, "sobaric_surface", 15) == 0) { return sc::isobaric_surface; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 's': if (memcmp(key + 1, "onde_correction", 15) == 0) { return sc::sonde_correction; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 17: if (memcmp(key + 0, "wind_gust_max_dir", 17) == 0) { return sc::wind_gust_max_dir; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 19: if (memcmp(key + 0, "wind_gust_max_speed", 19) == 0) { return sc::wind_gust_max_speed; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; case 22: if (memcmp(key + 0, "station_height_quality", 22) == 0) { return sc::station_height_quality; } else { wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } break; default: wreport::error_notfound::throwf("Shortcut name '%s' not found", key); } } } } dballe-8.6/dballe/core/shortcuts.h0000644000175000017500000000734213554564124014134 00000000000000#ifndef DBALLE_CORE_SHORTCUTS_H #define DBALLE_CORE_SHORTCUTS_H #include #include #include namespace dballe { namespace impl { struct Shortcut { bool station_data; Level level; Trange trange; wreport::Varcode code; static const Shortcut& by_name(const char* name); static const Shortcut& by_name(const std::string& name); static const Shortcut& by_name(const char* name, unsigned len); bool operator==(const Shortcut& o) const { return std::tie(station_data, level, trange, code) == std::tie(o.station_data, o.level, o.trange, o.code); } }; std::ostream& operator<<(std::ostream& out, const Shortcut& shortcut); namespace sc { extern const Shortcut st_type; extern const Shortcut st_name; extern const Shortcut st_name_icao; extern const Shortcut rep_memo; extern const Shortcut poll_lcode; extern const Shortcut poll_scode; extern const Shortcut poll_gemscode; extern const Shortcut poll_source; extern const Shortcut poll_atype; extern const Shortcut poll_ttype; extern const Shortcut flight_reg_no; extern const Shortcut flight_phase; extern const Shortcut flight_roll; extern const Shortcut navsys; extern const Shortcut data_relay; extern const Shortcut wind_inst; extern const Shortcut temp_precision; extern const Shortcut latlon_spec; extern const Shortcut timesig; extern const Shortcut block; extern const Shortcut station; extern const Shortcut ident; extern const Shortcut year; extern const Shortcut month; extern const Shortcut day; extern const Shortcut hour; extern const Shortcut minute; extern const Shortcut second; extern const Shortcut latitude; extern const Shortcut longitude; extern const Shortcut height_station; extern const Shortcut height_baro; extern const Shortcut height_release; extern const Shortcut station_height_quality; extern const Shortcut isobaric_surface; extern const Shortcut st_dir; extern const Shortcut st_speed; extern const Shortcut meas_equip_type; extern const Shortcut sonde_type; extern const Shortcut sonde_method; extern const Shortcut sonde_correction; extern const Shortcut sonde_tracking; extern const Shortcut press; extern const Shortcut press_3h; extern const Shortcut press_24h; extern const Shortcut water_temp; extern const Shortcut height_anem; extern const Shortcut press_tend; extern const Shortcut visibility; extern const Shortcut pres_wtr; extern const Shortcut past_wtr1_3h; extern const Shortcut past_wtr1_6h; extern const Shortcut past_wtr2_3h; extern const Shortcut past_wtr2_6h; extern const Shortcut metar_wtr; extern const Shortcut tot_prec1; extern const Shortcut tot_prec3; extern const Shortcut tot_prec6; extern const Shortcut tot_prec12; extern const Shortcut tot_prec24; extern const Shortcut tot_snow; extern const Shortcut state_ground; extern const Shortcut press_msl; extern const Shortcut qnh; extern const Shortcut temp_2m; extern const Shortcut wet_temp_2m; extern const Shortcut dewpoint_2m; extern const Shortcut humidity; extern const Shortcut wind_dir; extern const Shortcut wind_speed; extern const Shortcut wind_gust_max_speed; extern const Shortcut wind_gust_max_dir; extern const Shortcut ex_ccw_wind; extern const Shortcut ex_cw_wind; extern const Shortcut cloud_n; extern const Shortcut cloud_nh; extern const Shortcut cloud_hh; extern const Shortcut cloud_cl; extern const Shortcut cloud_cm; extern const Shortcut cloud_ch; extern const Shortcut cloud_n1; extern const Shortcut cloud_c1; extern const Shortcut cloud_h1; extern const Shortcut cloud_n2; extern const Shortcut cloud_c2; extern const Shortcut cloud_h2; extern const Shortcut cloud_n3; extern const Shortcut cloud_c3; extern const Shortcut cloud_h3; extern const Shortcut cloud_n4; extern const Shortcut cloud_c4; extern const Shortcut cloud_h4; } } } #endif dballe-8.6/dballe/core/var.cc0000644000175000017500000001207513554564112013020 00000000000000/* * dballe/var - DB-All.e specialisation of wreport variable * * Copyright (C) 2005,2010 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ #include "var.h" #include using namespace wreport; using namespace std; namespace dballe { /** * Format a varcode as fast as possible. * * It assumes res is at least 7 chars long. */ static const char* digits6bit = "00010203040506070809" "10111213141516171819" "20212223242526272829" "30313233343536373839" "40414243444546474849" "50515253545556575859" "60616263"; static const char* digits8bit = "000001002003004005006007008009" "010011012013014015016017018019" "020021022023024025026027028029" "030031032033034035036037038039" "040041042043044045046047048049" "050051052053054055056057058059" "060061062063064065066067068069" "070071072073074075076077078079" "080081082083084085086087088089" "090091092093094095096097098099" "100101102103104105106107108109" "110111112113114115116117118119" "120121122123124125126127128129" "130131132133134135136137138139" "140141142143144145146147148149" "150151152153154155156157158159" "160161162163164165166167168169" "170171172173174175176177178179" "180181182183184185186187188189" "190191192193194195196197198199" "200201202203204205206207208209" "210211212213214215216217218219" "220221222223224225226227228229" "230231232233234235236237238239" "240241242243244245246247248249" "250251252253254255"; static inline void format_xy(wreport::Varcode code, char* buf) { const char* src = digits6bit + (WR_VAR_X(code) & 0x3f) * 2; buf[1] = src[0]; buf[2] = src[1]; src = digits8bit + (WR_VAR_Y(code) & 0xff) * 3; buf[3] = src[0]; buf[4] = src[1]; buf[5] = src[2]; buf[6] = 0; } void format_code(wreport::Varcode code, char* buf) { // Format variable code switch (WR_VAR_F(code)) { case 0: buf[0] = 'B'; break; case 1: buf[0] = 'R'; break; case 2: buf[0] = 'C'; break; case 3: buf[0] = 'D'; break; default: buf[0] = '?'; break; } format_xy(code, buf); } void format_bcode(wreport::Varcode code, char* buf) { buf[0] = 'B'; format_xy(code, buf); } void resolve_varlist(const std::string& varlist, std::function dest) { if (varlist.empty()) throw error_consistency("cannot parse a Varcode list out of an empty string"); size_t beg = 0; while (true) { size_t end = varlist.find(',', beg); if (end == string::npos) { dest(resolve_varcode(varlist.substr(beg))); break; } else { dest(resolve_varcode(varlist.substr(beg, end-beg))); } beg = end + 1; } } void resolve_varlist(const std::string& varlist, std::set& out) { resolve_varlist(varlist, [&](wreport::Varcode code) { out.insert(code); }); } wreport::Varcode map_code_to_dballe(wreport::Varcode code) { switch (code) { case WR_VAR(0, 7, 1): return WR_VAR(0, 7, 30); case WR_VAR(0, 10, 3): return WR_VAR(0, 10, 8); case WR_VAR(0, 10, 61): return WR_VAR(0, 10, 60); case WR_VAR(0, 12, 1): return WR_VAR(0, 12, 101); case WR_VAR(0, 12, 2): return WR_VAR(0, 12, 102); case WR_VAR(0, 12, 3): return WR_VAR(0, 12, 103); default: return code; } } std::unique_ptr var_copy_without_unset_attrs(const wreport::Var& var) { unique_ptr copy(newvar(var.code())); copy->setval(var); // Copy value performing conversions for (const Var* a = var.next_attr(); a; a = a->next_attr()) { // Skip undefined attributes if (!a->isset()) continue; auto acopy = newvar(map_code_to_dballe(a->code())); acopy->setval(*a); copy->seta(move(acopy)); } return copy; } std::unique_ptr var_copy_without_unset_attrs( const wreport::Var& var, wreport::Varcode code) { unique_ptr copy(newvar(code)); copy->setval(var); // Copy value performing conversions for (const Var* a = var.next_attr(); a; a = a->next_attr()) { // Skip undefined attributes if (!a->isset()) continue; auto acopy = newvar(map_code_to_dballe(a->code())); acopy->setval(*a); copy->seta(move(acopy)); } return copy; } } dballe-8.6/dballe/core/values-test.cc0000644000175000017500000000052513554564112014501 00000000000000#include "tests.h" #include "values.h" #include using namespace std; using namespace dballe::tests; using namespace dballe; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_values"); void Tests::register_tests() { add_method("empty", []() { }); } } dballe-8.6/dballe/core/match-wreport.cc0000644000175000017500000001404313554564112015021 00000000000000#include #include #include #include #include #include using namespace wreport; using namespace std; namespace dballe { MatchedSubset::MatchedSubset(const Subset& r) : r(r), lat(MISSING_INT), lon(MISSING_INT), var_ana_id(0), var_block(0), var_station(0), var_rep_memo(0) { int ye = MISSING_INT, mo = MISSING_INT, da = MISSING_INT, ho = MISSING_INT, mi = MISSING_INT, se = MISSING_INT; // Scan message taking note of significant values for (Subset::const_iterator i = r.begin(); i != r.end(); ++i) { switch (i->code()) { case WR_VAR(0, 1, 1): var_block = &*i; break; case WR_VAR(0, 1, 2): var_station = &*i; break; case WR_VAR(0, 1, 192): var_ana_id = &*i; break; case WR_VAR(0, 1, 194): var_rep_memo = &*i; break; case WR_VAR(0, 4, 1): ye = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 2): mo = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 3): da = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 4): ho = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 5): mi = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 6): se = i->enq(MISSING_INT); break; case WR_VAR(0, 4, 7): se = i->isset() ? lround(i->enqd()) : MISSING_INT; break; case WR_VAR(0, 5, 1): case WR_VAR(0, 5, 2): if (i->isset()) lat = i->enqd() * 100000; break; case WR_VAR(0, 6, 1): case WR_VAR(0, 6, 2): if (i->isset()) lon = i->enqd() * 100000; break; } } // Fill in missing date bits date = Datetime::lower_bound(ye, mo, da, ho, mi, se); } MatchedSubset::~MatchedSubset() { } matcher::Result MatchedSubset::match_var_id(int val) const { for (Subset::const_iterator i = r.begin(); i != r.end(); ++i) { if (const Var* a = i->enqa(WR_VAR(0, 33, 195))) if (a->enqi() == val) return matcher::MATCH_YES; } return matcher::MATCH_NA; } matcher::Result MatchedSubset::match_station_id(int val) const { if (!var_ana_id) return matcher::MATCH_NA; if (!var_ana_id->isset()) return matcher::MATCH_NA; return var_ana_id->enqi() == val ? matcher::MATCH_YES : matcher::MATCH_NO; } matcher::Result MatchedSubset::match_station_wmo(int block, int station) const { if (!var_block) return matcher::MATCH_NA; if (!var_block->isset()) return matcher::MATCH_NA; if (var_block->enqi() != block) return matcher::MATCH_NO; if (station == -1) return matcher::MATCH_YES; if (!var_station) return matcher::MATCH_NA; if (!var_station->isset()) return matcher::MATCH_NA; if (var_station->enqi() != station) return matcher::MATCH_NO; return matcher::MATCH_YES; } matcher::Result MatchedSubset::match_datetime(const DatetimeRange& range) const { if (date.is_missing()) return matcher::MATCH_NA; return range.contains(date) ? matcher::MATCH_YES : matcher::MATCH_NO; } matcher::Result MatchedSubset::match_coords(const LatRange& latrange, const LonRange& lonrange) const { matcher::Result r1 = matcher::MATCH_NA; if (lat != MISSING_INT) r1 = latrange.contains(lat) ? matcher::MATCH_YES : matcher::MATCH_NO; else if (latrange.is_missing()) r1 = matcher::MATCH_YES; matcher::Result r2 = matcher::MATCH_NA; if (lon != MISSING_INT) r2 = lonrange.contains(lon) ? matcher::MATCH_YES : matcher::MATCH_NO; else if (lonrange.is_missing()) r2 = matcher::MATCH_YES; if (r1 == matcher::MATCH_YES && r2 == matcher::MATCH_YES) return matcher::MATCH_YES; if (r1 == matcher::MATCH_NO || r2 == matcher::MATCH_NO) return matcher::MATCH_NO; return matcher::MATCH_NA; } matcher::Result MatchedSubset::match_rep_memo(const char* memo) const { if (!var_rep_memo) return matcher::MATCH_NA; if (!var_rep_memo->isset()) return matcher::MATCH_NA; return strcmp(memo, var_rep_memo->enqc()) == 0 ? matcher::MATCH_YES : matcher::MATCH_NO; } MatchedBulletin::MatchedBulletin(const wreport::Bulletin& r) : r(r) { subsets = new const MatchedSubset*[r.subsets.size()]; for (unsigned i = 0; i < r.subsets.size(); ++i) subsets[i] = new MatchedSubset(r.subsets[i]); } MatchedBulletin::~MatchedBulletin() { for (unsigned i = 0; i < r.subsets.size(); ++i) delete subsets[i]; delete[] subsets; } matcher::Result MatchedBulletin::match_var_id(int val) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_var_id(val) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedBulletin::match_station_id(int val) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_station_id(val) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedBulletin::match_station_wmo(int block, int station) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_station_wmo(block, station) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedBulletin::match_datetime(const DatetimeRange& range) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_datetime(range) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedBulletin::match_coords(const LatRange& latrange, const LonRange& lonrange) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_coords(latrange, lonrange) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedBulletin::match_rep_memo(const char* memo) const { for (unsigned i = 0; i < r.subsets.size(); ++i) if (subsets[i]->match_rep_memo(memo) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } } dballe-8.6/dballe/core/arrayfile.cc0000644000175000017500000000124113554564112014177 00000000000000#include "arrayfile.h" using namespace std; namespace dballe { namespace core { ArrayFile::ArrayFile(Encoding type) : File("array", NULL, false), file_type(type), current(0) { } ArrayFile::~ArrayFile() { } Encoding ArrayFile::encoding() const { return file_type; } void ArrayFile::write(const std::string& msg) { msgs.push_back(BinaryMessage(file_type)); msgs.back().data = msg; msgs.back().pathname = m_name; msgs.back().offset = msgs.size() - 1; msgs.back().index = msgs.size() - 1; } BinaryMessage ArrayFile::read() { if (current >= msgs.size()) return BinaryMessage(file_type); else return msgs[current++]; } } } dballe-8.6/dballe/core/query.h0000644000175000017500000001356513572423710013242 00000000000000#ifndef DBALLE_CORE_QUERY_H #define DBALLE_CORE_QUERY_H #include #include #include #include #include /** * Values for query modifier flags */ /** When values from different reports exist on the same point, only report the * one from the report with the highest priority */ #define DBA_DB_MODIFIER_BEST (1 << 0) /** Do not bother sorting the results */ #define DBA_DB_MODIFIER_UNSORTED (1 << 5) /** Sort by report after ana_id, to ease reconstructing messages on export */ #define DBA_DB_MODIFIER_SORT_FOR_EXPORT (1 << 7) /// Add minimum date, maximum date and data count details to summary query results #define DBA_DB_MODIFIER_SUMMARY_DETAILS (1 << 8) /// Also get attributes alongside data #define DBA_DB_MODIFIER_WITH_ATTRIBUTES (1 << 9) namespace dballe { namespace core { struct JSONWriter; /// Standard dballe::Query implementation struct Query : public dballe::Query { static const uint32_t WANT_MISSING_IDENT = (1 << 0); static const uint32_t WANT_MISSING_LTYPE1 = (1 << 1); static const uint32_t WANT_MISSING_L1 = (1 << 2); static const uint32_t WANT_MISSING_LTYPE2 = (1 << 3); static const uint32_t WANT_MISSING_L2 = (1 << 4); static const uint32_t WANT_MISSING_PIND = (1 << 5); static const uint32_t WANT_MISSING_P1 = (1 << 6); static const uint32_t WANT_MISSING_P2 = (1 << 7); /** * Set a bit a 1 with WANT_MISSING_* constants to specify that the query * wants results in which the corresponding field is set to a missing * value. */ uint32_t want_missing = 0; int ana_id = MISSING_INT; int priomin = MISSING_INT; int priomax = MISSING_INT; std::string report; int mobile = MISSING_INT; Ident ident; LatRange latrange; LonRange lonrange; DatetimeRange dtrange; Level level; Trange trange; std::set varcodes; std::string query; std::string ana_filter; std::string data_filter; std::string attr_filter; int limit = MISSING_INT; int block = MISSING_INT; int station = MISSING_INT; bool operator==(const Query& o) const { return std::tie(want_missing, ana_id, priomin, priomax, report, mobile, ident, latrange, lonrange, dtrange, level, trange, varcodes, query, ana_filter, data_filter, attr_filter, limit, block, station) == std::tie(o.want_missing, o.ana_id, o.priomin, o.priomax, o.report, o.mobile, o.ident, o.latrange, o.lonrange, o.dtrange, o.level, o.trange, o.varcodes, o.query, o.ana_filter, o.data_filter, o.attr_filter, o.limit, o.block, o.station); } /** * Check the query fields for consistency, and fill in missing values: * * - month without year, day without month, and so on, cause errors * - only one longitude extreme without the other causes error * - min and max datetimes are filled with the actual minimum and maximum * values acceptable for that range (year=2017, for example, becomes * min=2017-01-01 00:00:00, max=2017-12-31 23:59:59 */ void validate(); std::unique_ptr clone() const override; unsigned get_modifiers() const; DatetimeRange get_datetimerange() const override { return dtrange; } void set_datetimerange(const DatetimeRange& dt) override { dtrange = dt; } LatRange get_latrange() const override { return latrange; } void set_latrange(const LatRange& lr) override { latrange = lr; } LonRange get_lonrange() const override { return lonrange; } void set_lonrange(const LonRange& lr) override { lonrange = lr; } Level get_level() const override { return level; } void set_level(const Level& level) override { this->level = level; } Trange get_trange() const override { return trange; } void set_trange(const Trange& trange) override { this->trange = trange; } void clear() override; /** * Set a value in the record according to an assignment encoded in a string. * * String can use keywords, aliases and varcodes. Examples: ana_id=3, * name=Bologna, B12012=32.4 * * In case of numeric parameter, a hyphen ("-") means MISSING_INT (e.g., * `leveltype2=-`). * * @param str * The string containing the assignment. */ void set_from_string(const char* str); /** * Set a record from a ", "-separated string of assignments. * * The implementation is not efficient and the method is not safe for any * input, since ", " could appear in a station identifier. It is however * useful to quickly create test queries for unit testing. */ void set_from_test_string(const std::string& s); /** * Return true if this query matches a subset of the given query. * * In other words, it returns true if this query is the same as \a other, * plus zero or more extra fields set, or zero or more ranges narrowed. */ bool is_subquery(const dballe::Query& other) const override; /// Print the query contents to stderr void print(FILE* out) const override; /// Send the contents to a JSONWriter void serialize(JSONWriter& out) const; /** * Parse the modifiers specification given a query=* string, returning the ORed * flags. */ static unsigned parse_modifiers(const char* str); /** * Return a reference to query downcasted as core::Query. * * Throws an exception if query is not a core::Query. */ static const Query& downcast(const dballe::Query& query); /** * Return a reference to query downcasted as core::Query. * * Throws an exception if query is not a core::Query. */ static Query& downcast(dballe::Query& query); static Query from_json(core::json::Stream& in); protected: void setf(const char* key, unsigned len, const char* val); void unset(const char* key, unsigned len); }; } } #endif dballe-8.6/dballe/core/file.h0000644000175000017500000000361513554564112013011 00000000000000#ifndef DBA_CORE_FILE_H #define DBA_CORE_FILE_H #include #include #include #include #include #include namespace dballe { namespace core { /// Base for dballe::File implementations class File : public dballe::File { protected: /// Name of the file std::string m_name; /// FILE structure used to read or write to the file FILE* fd; /// True if fd should be closed on destruction bool close_on_exit; /// Index of the last message read from the file or written to the file int idx; public: File(const std::string& name, FILE* fd, bool close_on_exit=true); virtual ~File(); std::string pathname() const override { return m_name; } void close() override; bool foreach(std::function dest) override; /** * Resolve the location of a test data file * * This should only be used during dballe unit tests. */ static std::string resolve_test_data_file(const std::string& name); /** * Open a test data file. * * This should only be used during dballe unit tests. */ static std::unique_ptr open_test_data_file(Encoding type, const std::string& name); }; class BufrFile : public dballe::core::File { public: BufrFile(const std::string& name, FILE* fd, bool close_on_exit=true) : File(name, fd, close_on_exit) {} Encoding encoding() const override { return Encoding::BUFR; } BinaryMessage read() override; void write(const std::string& msg) override; }; class CrexFile : public dballe::core::File { public: CrexFile(const std::string& name, FILE* fd, bool close_on_exit=true) : File(name, fd, close_on_exit) {} Encoding encoding() const override { return Encoding::CREX; } BinaryMessage read() override; void write(const std::string& msg) override; }; } } #endif dballe-8.6/dballe/core/json.cc0000644000175000017500000004150613554564112013202 00000000000000#include "json.h" #include "dballe/values.h" #include #include using namespace std; namespace dballe { namespace core { JSONWriter::JSONWriter(std::ostream& out) : out(out) {} JSONWriter::~JSONWriter() {} void JSONWriter::reset() { stack.clear(); } void JSONWriter::val_head() { if (!stack.empty()) { switch (stack.back()) { case LIST_FIRST: stack.back() = LIST; break; case LIST: out << ','; break; case MAPPING_KEY_FIRST: stack.back() = MAPPING_VAL; break; case MAPPING_KEY: out << ','; stack.back() = MAPPING_VAL; break; case MAPPING_VAL: out << ':'; stack.back() = MAPPING_KEY; break; } } } void JSONWriter::add_null() { val_head(); out << "null"; } void JSONWriter::add_bool(bool val) { val_head(); out << (val ? "true" : "false" ); } void JSONWriter::add_int(int val) { val_head(); out << to_string(val); } void JSONWriter::add_double(double val) { val_head(); double vint, vfrac; vfrac = modf(val, &vint); if (vfrac == 0.0) { out << to_string((int)vint); out << ".0"; } else out << to_string(val); } void JSONWriter::add_cstring(const char* val) { val_head(); out << '"'; for ( ; *val; ++val) switch (*val) { case '"': out << "\\\""; break; case '\\': out << "\\\\"; break; case '\b': out << "\\b"; break; case '\f': out << "\\f"; break; case '\n': out << "\\n"; break; case '\r': out << "\\r"; break; case '\t': out << "\\t"; break; default: out << *val; break; } out << '"'; } void JSONWriter::add_string(const std::string& val) { add_cstring(val.c_str()); } void JSONWriter::add_level(const Level& val) { start_list(); if (val.ltype1 != MISSING_INT) add(val.ltype1); else add_null(); if (val.l1 != MISSING_INT) add(val.l1); else add_null(); if (val.ltype2 != MISSING_INT) add(val.ltype2); else add_null(); if (val.l2 != MISSING_INT) add(val.l2); else add_null(); end_list(); } void JSONWriter::add_trange(const Trange& val) { start_list(); if (val.pind != MISSING_INT) add(val.pind); else add_null(); if (val.p1 != MISSING_INT) add(val.p1); else add_null(); if (val.p2 != MISSING_INT) add(val.p2); else add_null(); end_list(); } void JSONWriter::add_coords(const Coords& val) { start_list(); if (val.lat != MISSING_INT) add(val.lat); else add_null(); if (val.lon != MISSING_INT) add(val.lon); else add_null(); end_list(); } void JSONWriter::add_station(const Station& s) { start_mapping(); add("r", s.report); add("c", s.coords); if (s.ident) add("i", s.ident); end_mapping(); } void JSONWriter::add_dbstation(const DBStation& s) { start_mapping(); if (s.id != MISSING_INT) add("id", s.id); add("r", s.report); add("c", s.coords); if (s.ident) add("i", s.ident); end_mapping(); } void JSONWriter::add_values(const Values& values) { start_mapping(); for (const auto& var: values) { add(wreport::varcode_format(var->code())); start_mapping(); add("v"); add(*var); if (var->next_attr()) { add("a"); start_mapping(); for (const wreport::Var* attr = var->next_attr(); attr; attr = attr->next_attr()) { add(wreport::varcode_format(attr->code())); add(*attr); } end_mapping(); } end_mapping(); } end_mapping(); } void JSONWriter::add_dbvalues(const DBValues& values) { } void JSONWriter::add_datetime(const Datetime& val) { if (val.is_missing()) add_null(); else { start_list(); if (val.year != MISSING_INT) add(val.year); else add_null(); if (val.month != MISSING_INT) add(val.month); else add_null(); if (val.day != MISSING_INT) add(val.day); else add_null(); if (val.hour != MISSING_INT) add(val.hour); else add_null(); if (val.minute != MISSING_INT) add(val.minute); else add_null(); if (val.second != MISSING_INT) add(val.second); else add_null(); end_list(); } } void JSONWriter::add_datetimerange(const DatetimeRange& val) { start_list(); add(val.min); add(val.max); end_list(); } void JSONWriter::add_number(const std::string& val) { val_head(); out << val; } void JSONWriter::add_var(const wreport::Var& val) { if (val.isset()) { if (val.info()->type == wreport::Vartype::String || val.info()->type == wreport::Vartype::Binary) { add_string(val.format()); } else { add_number(val.format()); } } else { add_null(); } } void JSONWriter::add_ident(const Ident& val) { if (val.is_missing()) add_null(); else add_cstring(val); } void JSONWriter::add_break() { out << "\n"; } void JSONWriter::start_list() { val_head(); out << '['; stack.push_back(LIST_FIRST); } void JSONWriter::end_list() { out << ']'; stack.pop_back(); } void JSONWriter::start_mapping() { val_head(); out << '{'; stack.push_back(MAPPING_KEY_FIRST); } void JSONWriter::end_mapping() { out << '}'; stack.pop_back(); } namespace json { void Stream::expect_token(const char* token) { for (const char* s = token; *s; ++s) { int c = in.get(); if (c != *s) { if (c == EOF) // throw JSONParseException(str::fmtf("end of file reached looking for %s in %s", s, expected)); throw JSONParseException("unexpected end of file reached"); else // throw JSONParseException(str::fmtf("unexpected character '%c' looking for %s in %s", c, s, expected)); throw JSONParseException("unexpected character"); } } } void Stream::skip_spaces() { while (isspace(in.peek())) in.get(); } double Stream::parse_double() { string val; std::tie(val, std::ignore) = parse_number(); return std::stod(val); } std::tuple Stream::parse_number() { string num; bool done = false; bool is_double = false; while (!done) { int c = in.peek(); switch (c) { case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': num.append(1, in.get()); break; case '.': case 'e': case 'E': case '+': is_double = true; num.append(1, in.get()); break; default: done = true; } } skip_spaces(); return std::make_tuple(num, is_double); } std::string Stream::parse_string() { string res; int c = in.get(); // Eat the leading '"' if (c != '"') throw JSONParseException("expected string does not begin with '\"'"); bool done = false; while (!done) { int c = in.get(); switch (c) { case '\\': c = in.get(); if (c == EOF) throw JSONParseException("unterminated string"); switch (c) { case 'b': res.append(1, '\b'); break; case 'f': res.append(1, '\f'); break; case 'n': res.append(1, '\n'); break; case 'r': res.append(1, '\r'); break; case 't': res.append(1, '\t'); break; default: res.append(1, c); break; } break; case '"': done = true; break; case EOF: throw JSONParseException("unterminated string"); default: res.append(1, c); break; } } skip_spaces(); return res; } Coords Stream::parse_coords() { Coords res; unsigned idx = 0; parse_array([&]{ switch (identify_next()) { case JSON_NUMBER: switch (idx) { case 0: res.lat = parse_signed(); break; case 1: res.lon = parse_signed(); break; default: throw JSONParseException("extra element in Coords array"); } break; case JSON_NULL: expect_token("null"); break; default: throw JSONParseException("unexpected element in Coords array"); } ++idx; }); return res; } Ident Stream::parse_ident() { switch (identify_next()) { case JSON_STRING: return Ident(parse_string()); case JSON_NULL: expect_token("null"); return Ident(); default: throw JSONParseException("unexpected element for Ident"); } } Station Stream::parse_station() { Station res; parse_object([&](const std::string& key) { if (key == "r") res.report = parse_string(); else if (key == "c") res.coords = parse_coords(); else if (key == "i") res.ident = parse_ident(); else throw core::JSONParseException("unsupported key \"" + key + "\" for Station"); }); return res; } DBStation Stream::parse_dbstation() { DBStation res; parse_object([&](const std::string& key) { if (key == "id") res.id = parse_unsigned(); else if (key == "r") res.report = parse_string(); else if (key == "c") res.coords = parse_coords(); else if (key == "i") res.ident = parse_ident(); else throw core::JSONParseException("unsupported key \"" + key + "\" for DBStation"); }); return res; } Level Stream::parse_level() { Level res; unsigned idx = 0; parse_array([&]{ switch (identify_next()) { case JSON_NUMBER: switch (idx) { case 0: res.ltype1 = parse_signed(); break; case 1: res.l1 = parse_signed(); break; case 2: res.ltype2 = parse_signed(); break; case 3: res.l2 = parse_signed(); break; default: throw JSONParseException("extra element in Level array"); } break; case JSON_NULL: expect_token("null"); break; default: throw JSONParseException("unexpected element in Level array"); } ++idx; }); return res; } Trange Stream::parse_trange() { Trange res; unsigned idx = 0; parse_array([&]{ switch (identify_next()) { case JSON_NUMBER: switch (idx) { case 0: res.pind = parse_signed(); break; case 1: res.p1 = parse_signed(); break; case 2: res.p2 = parse_signed(); break; default: throw JSONParseException("extra element in Trange array"); } break; case JSON_NULL: expect_token("null"); break; default: throw JSONParseException("unexpected element in Trange array"); } ++idx; }); return res; } Datetime Stream::parse_datetime() { switch (identify_next()) { case JSON_NULL: expect_token("null"); return Datetime(); case JSON_ARRAY: { Datetime res; unsigned idx = 0; parse_array([&]{ switch (identify_next()) { case JSON_NUMBER: switch (idx) { case 0: res.year = parse_unsigned(); break; case 1: res.month = parse_signed(); break; case 2: res.day = parse_signed(); break; case 3: res.hour = parse_signed(); break; case 4: res.minute = parse_signed(); break; case 5: res.second = parse_signed(); break; default: throw JSONParseException("extra element in Datetime array"); } break; case JSON_NULL: expect_token("null"); break; default: throw JSONParseException("unexpected element in Datetime array"); } ++idx; }); return res; } default: throw JSONParseException("unexpected element for Datetime"); } } DatetimeRange Stream::parse_datetimerange() { DatetimeRange res; unsigned idx = 0; parse_array([&]{ switch (idx) { case 0: res.min = parse_datetime(); break; case 1: res.max = parse_datetime(); break; default: throw JSONParseException("extra element in DatetimeRange array"); } ++idx; }); return res; } void Stream::parse_array(std::function on_element) { if (in.get() != '[') throw JSONParseException("expected array does not begin with '['"); skip_spaces(); while (in.peek() != ']') { on_element(); if (in.peek() == ',') in.get(); skip_spaces(); } if (in.get() != ']') throw JSONParseException("array does not end with '['"); skip_spaces(); } void Stream::parse_object(std::function on_value) { if (in.get() != '{') throw JSONParseException("expected object does not begin with '{'"); skip_spaces(); while (in.peek() != '}') { if (in.peek() != '"') throw JSONParseException("expected a string as object key"); std::string key = parse_string(); skip_spaces(); if (in.peek() == ':') in.get(); else throw JSONParseException("':' expected after object key"); on_value(key); if (in.peek() == ',') in.get(); skip_spaces(); } if (in.get() != '}') throw JSONParseException("expected object does not end with '}'"); skip_spaces(); } Element Stream::identify_next() { skip_spaces(); switch (in.peek()) { case EOF: throw JSONParseException("JSON string is truncated"); case '{': return JSON_OBJECT; case '[': return JSON_ARRAY; case '"': return JSON_STRING; case '-': case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': return JSON_NUMBER; case 't': return JSON_TRUE; case 'f': return JSON_FALSE; case 'n': return JSON_NULL; default: // throw JSONParseException(str::fmtf("unexpected character '%c'", in.in.peek())); throw JSONParseException("unexpected character"); } } } static void parse_value(json::Stream& in, JSONReader& e) { switch (in.identify_next()) { case json::JSON_OBJECT: e.on_start_mapping(); in.parse_object([&](const std::string& key) { e.on_add_string(key); parse_value(in, e); }); e.on_end_mapping(); break; case json::JSON_ARRAY: e.on_start_list(); in.parse_array([&]{ parse_value(in, e); }); e.on_end_list(); break; case json::JSON_STRING: e.on_add_string(in.parse_string()); break; case json::JSON_NUMBER: { std::string val; bool is_double; std::tie(val, is_double) = in.parse_number(); if (is_double) e.on_add_double(std::stod(val)); else e.on_add_int(stoi(val)); break; } case json::JSON_TRUE: in.expect_token("true"); e.on_add_bool(true); in.skip_spaces(); break; case json::JSON_FALSE: in.expect_token("false"); e.on_add_bool(false); in.skip_spaces(); break; case json::JSON_NULL: in.expect_token("null"); e.on_add_null(); in.skip_spaces(); break; default: // throw JSONParseException(str::fmtf("unexpected character '%c'", in.in.peek())); throw JSONParseException("unexpected character"); } in.skip_spaces(); } void JSONReader::parse(std::istream& in) { json::Stream jstream(in); parse_value(jstream, *this); } } } dballe-8.6/dballe/core/string-test.cc0000644000175000017500000000470013554564112014507 00000000000000#include "dballe/core/tests.h" #include "string.h" using namespace dballe; using namespace wreport::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_string"); void Tests::register_tests() { add_method("url_pop_query_string", []() { string url = "http://example.org"; string res; wassert_false(url_pop_query_string(url, "foo", res)); wassert(actual(res) == ""); wassert(actual(url) == "http://example.org"); url = "http://example.org?foo=bar"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "bar"); wassert(actual(url) == "http://example.org"); url = "http://example.org?foo=bar&baz=gnu"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "bar"); wassert(actual(url) == "http://example.org?baz=gnu"); url = "http://example.org?baz=gnu&foo=bar"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "bar"); wassert(actual(url) == "http://example.org?baz=gnu"); url = "http://example.org?baz=gnu&foo=bar&wibble=wobble"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "bar"); wassert(actual(url) == "http://example.org?baz=gnu&wibble=wobble"); url = "http://example.org?foo=bar&foo=baz"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "bar"); wassert(actual(url) == "http://example.org?foo=baz"); url = "http://example.org?foobar=bar&foo=baz"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == "baz"); wassert(actual(url) == "http://example.org?foobar=bar"); url = "http://example.org?foo&bar=baz"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == ""); wassert(actual(url) == "http://example.org?bar=baz"); url = "http://example.org?bar=baz&foo"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == ""); wassert(actual(url) == "http://example.org?bar=baz"); url = "http://example.org?foo&"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == ""); wassert(actual(url) == "http://example.org"); url = "http://example.org?foo=&"; wassert_true(url_pop_query_string(url, "foo", res)); wassert(actual(res) == ""); wassert(actual(url) == "http://example.org"); }); } } dballe-8.6/dballe/core/defs-test.cc0000644000175000017500000000043613554564112014124 00000000000000#include "tests.h" #include "defs.h" using namespace dballe; using namespace wreport::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_defs"); void Tests::register_tests() { } } dballe-8.6/dballe/core/varmatch.cc0000644000175000017500000001327013554564112014033 00000000000000/* * core/varmatch - Variable matcher * * Copyright (C) 2013 ARPA-SIM * * This program is free software; you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation; either version 2 of the License. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program; if not, write to the Free Software * Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA * * Author: Enrico Zini */ #include "varmatch.h" #include "dballe/var.h" #include #include #include using namespace std; using namespace wreport; namespace dballe { Varmatch::Varmatch(wreport::Varcode code) : code(code) {} bool Varmatch::operator()(const wreport::Var& var) const { return var.code() == code; } namespace varmatch { struct BetweenInt : public Varmatch { int min, max; BetweenInt(Varcode code, int min, int max) : Varmatch(code), min(min), max(max) {} bool operator()(const wreport::Var& var) const { if (!Varmatch::operator()(var)) return false; if (!var.isset()) return false; int ival = var.enqi(); return min <= ival && ival <= max; } }; struct BetweenString : public Varmatch { string min, max; BetweenString(Varcode code, const std::string& min, const std::string& max) : Varmatch(code), min(min), max(max) {} bool operator()(const wreport::Var& var) const { if (!Varmatch::operator()(var)) return false; if (!var.isset()) return false; string sval = var.enqs(); return min <= sval && sval <= max; } }; template struct Op : public Varmatch { OP op; T val; Op(Varcode code, const T& val) : Varmatch(code), val(val) {} bool operator()(const wreport::Var& var) const { if (!Varmatch::operator()(var)) return false; if (!var.isset()) return false; return op(var.enq(), val); } }; template static unique_ptr make_op(Varcode code, const std::string& op, const T& val) { if (op == "<") return unique_ptr(new Op< T, less >(code, val)); else if (op == "<=") return unique_ptr(new Op< T, less_equal >(code, val)); else if (op == ">") return unique_ptr(new Op< T, greater >(code, val)); else if (op == ">=") return unique_ptr(new Op< T, greater_equal >(code, val)); else if (op == "=" || op == "==") return unique_ptr(new Op< T, equal_to >(code, val)); else if (op == "<>") return unique_ptr(new Op< T, not_equal_to >(code, val)); else error_consistency::throwf("cannot understand comparison operator '%s'", op.c_str()); } } std::unique_ptr Varmatch::parse(const std::string& filter) { size_t sep1_begin = filter.find_first_of("<=>"); if (sep1_begin == string::npos) error_consistency::throwf("cannot find any operator in filter '%s'", filter.c_str()); size_t sep1_end = filter.find_first_not_of("<=>", sep1_begin); if (sep1_end == string::npos) error_consistency::throwf("cannot find end of first operator in filter '%s'", filter.c_str()); size_t sep2_begin = filter.find_first_of("<=>", sep1_end); if (sep2_begin != string::npos) { // min<=B12345<=max size_t sep2_end = filter.find_first_not_of("<=>", sep2_begin); if (sep2_end == string::npos) error_consistency::throwf("cannot find end of second operator in filter '%s'", filter.c_str()); Varcode code = resolve_varcode(filter.substr(sep1_end, sep2_begin - sep1_end).c_str()); Varinfo info = varinfo(code); string min = filter.substr(0, sep1_begin); string max = filter.substr(sep2_end); switch (info->type) { case Vartype::String: return unique_ptr(new varmatch::BetweenString(code, min, max)); case Vartype::Binary: error_consistency::throwf("cannot apply filter '%s' to a binary variable", filter.c_str()); case Vartype::Integer: case Vartype::Decimal: { int imin = info->encode_decimal(strtod(min.c_str(), NULL)); int imax = info->encode_decimal(strtod(max.c_str(), NULL)); return unique_ptr(new varmatch::BetweenInt(code, imin, imax)); } } error_consistency::throwf("unsupported variable type %d", (int)info->type); } else { // B12345<=>val Varcode code = resolve_varcode(filter.substr(0, sep1_begin).c_str()); Varinfo info = varinfo(code); string op = filter.substr(sep1_begin, sep1_end - sep1_begin); switch (info->type) { case Vartype::String: return varmatch::make_op(code, op, filter.substr(sep1_end)); case Vartype::Binary: error_consistency::throwf("cannot apply filter '%s' to a binary variable", filter.c_str()); case Vartype::Integer: case Vartype::Decimal: { int val = info->encode_decimal(strtod(filter.substr(sep1_end).c_str(), NULL)); return varmatch::make_op(code, op, val); } } error_consistency::throwf("unsupported variable type %d", (int)info->type); } } } dballe-8.6/dballe/core/tests.cc0000644000175000017500000001657113554564112013377 00000000000000#include "tests.h" #include "matcher.h" #include #include #include #include #include using namespace wreport; using namespace dballe::tests; using namespace std; namespace dballe { namespace tests { static std::string tag; const static Varcode generator_varcodes[] = { WR_VAR(0, 1, 1), WR_VAR(0, 1, 2), WR_VAR(0, 1, 8), WR_VAR(0, 1, 11), WR_VAR(0, 1, 12), WR_VAR(0, 1, 13), WR_VAR(0, 2, 1), WR_VAR(0, 2, 2), WR_VAR(0, 2, 5), WR_VAR(0, 2, 11), WR_VAR(0, 2, 12), WR_VAR(0, 2, 61), WR_VAR(0, 2, 62), WR_VAR(0, 2, 63), WR_VAR(0, 2, 70), WR_VAR(0, 4, 1), WR_VAR(0, 4, 2), WR_VAR(0, 4, 3), WR_VAR(0, 4, 4), WR_VAR(0, 4, 5), WR_VAR(0, 5, 1), WR_VAR(0, 6, 1), WR_VAR(0, 7, 30), WR_VAR(0, 7, 2), WR_VAR(0, 7, 31), WR_VAR(0, 8, 1), WR_VAR(0, 8, 4), WR_VAR(0, 8, 21), WR_VAR(0, 10, 3), WR_VAR(0, 10, 4), WR_VAR(0, 10, 51), WR_VAR(0, 10, 61), WR_VAR(0, 10, 63), WR_VAR(0, 10, 197), WR_VAR(0, 11, 1), WR_VAR(0, 11, 2), WR_VAR(0, 11, 3), WR_VAR(0, 11, 4), }; #if 0 generator::~generator() { for (std::vector::iterator i = reused_pseudoana_fixed.begin(); i != reused_pseudoana_fixed.end(); i++) dba_record_delete(*i); for (std::vector::iterator i = reused_pseudoana_mobile.begin(); i != reused_pseudoana_mobile.end(); i++) dba_record_delete(*i); for (std::vector::iterator i = reused_context.begin(); i != reused_context.end(); i++) dba_record_delete(*i); } dba_err generator::fill_pseudoana(dba_record rec, bool mobile) { dba_record ana; if ((mobile && reused_pseudoana_mobile.empty()) || (!mobile && reused_pseudoana_fixed.empty()) || rnd(0.3)) { DBA_RUN_OR_RETURN(dba_record_create(&ana)); /* Pseudoana */ DBA_RUN_OR_RETURN(dba_record_key_setd(ana, DBA_KEY_LAT, rnd(-90, 90))); DBA_RUN_OR_RETURN(dba_record_key_setd(ana, DBA_KEY_LON, rnd(-180, 180))); if (mobile) { DBA_RUN_OR_RETURN(dba_record_key_setc(ana, DBA_KEY_IDENT, rnd(10).c_str())); DBA_RUN_OR_RETURN(dba_record_key_seti(ana, DBA_KEY_MOBILE, 1)); reused_pseudoana_mobile.push_back(ana); } else { //DBA_RUN_OR_RETURN(dba_record_key_seti(ana, DBA_KEY_BLOCK, rnd(0, 99))); //DBA_RUN_OR_RETURN(dba_record_key_seti(ana, DBA_KEY_STATION, rnd(0, 999))); DBA_RUN_OR_RETURN(dba_record_key_seti(ana, DBA_KEY_MOBILE, 0)); reused_pseudoana_fixed.push_back(ana); } } else { if (mobile) ana = reused_pseudoana_mobile[rnd(0, reused_pseudoana_mobile.size() - 1)]; else ana = reused_pseudoana_fixed[rnd(0, reused_pseudoana_fixed.size() - 1)]; } DBA_RUN_OR_RETURN(dba_record_add(rec, ana)); return dba_error_ok(); } dba_err generator::fill_context(dba_record rec) { dba_record ctx; if (reused_context.empty() || rnd(0.7)) { DBA_RUN_OR_RETURN(dba_record_create(&ctx)); /* Context */ DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_YEAR, rnd(2002, 2005))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_MONTH, rnd(1, 12))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_DAY, rnd(1, 28))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_HOUR, rnd(0, 23))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_MIN, rnd(0, 59))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_LEVELTYPE1, rnd(0, 300))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_L1, rnd(0, 100000))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_LEVELTYPE2, rnd(0, 300))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_L2, rnd(0, 100000))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_PINDICATOR, rnd(0, 300))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_P1, rnd(0, 100000))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_P2, rnd(0, 100000))); DBA_RUN_OR_RETURN(dba_record_key_seti(ctx, DBA_KEY_REP_COD, rnd(105, 149))); reused_context.push_back(ctx); } else { ctx = reused_context[rnd(0, reused_context.size() - 1)]; } DBA_RUN_OR_RETURN(dba_record_add(rec, ctx)); return dba_error_ok(); } dba_err generator::fill_record(dba_record rec) { DBA_RUN_OR_RETURN(fill_pseudoana(rec, rnd(0.8))); DBA_RUN_OR_RETURN(fill_context(rec)); DBA_RUN_OR_RETURN(dba_record_var_setc(rec, generator_varcodes[rnd(0, sizeof(generator_varcodes) / sizeof(dba_varcode))], "1")); return dba_error_ok(); } #endif #if 0 dba_err read_file(dba_encoding type, const std::string& name, dba_raw_consumer& cons) { dba_err err = DBA_OK; dba_file file = open_test_data(name.c_str(), type); dba_rawmsg raw = 0; int found; DBA_RUN_OR_GOTO(cleanup, dba_rawmsg_create(&raw)); DBA_RUN_OR_GOTO(cleanup, dba_file_read(file, raw, &found)); while (found) { DBA_RUN_OR_GOTO(cleanup, cons.consume(raw)); DBA_RUN_OR_GOTO(cleanup, dba_file_read(file, raw, &found)); } cleanup: if (file) dba_file_delete(file); if (raw) dba_rawmsg_delete(raw); return err == DBA_OK ? dba_error_ok() : err; } #endif std::string datafile(const std::string& fname) { // Skip appending the test data path for pathnames starting with ./ if (fname[0] == '.') return fname; const char* testdatadirenv = getenv("DBA_TESTDATA"); std::string testdatadir = testdatadirenv ? testdatadirenv : "."; return testdatadir + "/" + fname; } unique_ptr open_test_data(const char* filename, Encoding type) { return unique_ptr(File::create(type, datafile(filename), "r")); } BinaryMessage read_rawmsg(const char* filename, Encoding type) { unique_ptr f = wcallchecked(open_test_data(filename, type)); BinaryMessage res = f->read(); wassert(actual(res).istrue()); return res; } #if 0 void vars_equal(const Record& expected) { return TestRecordVarsEqual(this->actual, expected); } void TestRecordValEqual::check() const { const wreport::Var* evar = expected.get(name); const wreport::Var* avar = actual.get(name); if (with_missing_int && evar && evar->enq(MISSING_INT) == MISSING_INT) evar = NULL; if (with_missing_int && avar && avar->enq(MISSING_INT) == MISSING_INT) avar = NULL; if (!evar && !avar) return; if (!evar || !avar || evar->code() != avar->code() || !evar->value_equals(*avar)) { std::stringstream ss; ss << "records differ on " << name << ": "; if (!evar) ss << "it is expected unset"; else ss << evar->format() << " is expected"; ss << ", but actual "; if (!avar) ss << "is unset"; else ss << "has " << avar->format(); throw TestFailed(ss.str()); } } #endif unique_ptr query_from_string(const std::string& s) { core::Query* q; unique_ptr res(q = new core::Query); q->set_from_test_string(s); return res; } core::Query core_query_from_string(const std::string& s) { core::Query q; q.set_from_test_string(s); return q; } void ActualMatcherResult::operator==(int expected) const { if (expected == _actual) return; std::stringstream ss; ss << "actual match result is " << matcher::result_format((matcher::Result)_actual) << " but it should be " << matcher::result_format((matcher::Result)expected); throw TestFailed(ss.str()); } void ActualMatcherResult::operator!=(int expected) const { if (expected != _actual) return; std::stringstream ss; ss << "actual match result is " << matcher::result_format((matcher::Result)_actual) << " but it should not be"; throw TestFailed(ss.str()); } } } dballe-8.6/dballe/core/var-test.cc0000644000175000017500000000325413554564112013774 00000000000000#include "tests.h" #include "var.h" using namespace dballe; using namespace wreport; using namespace wreport::tests; using namespace std; namespace tut { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("core_var"); void Tests::register_tests() { add_method("single", []() { set codes; resolve_varlist("B12101", codes); wassert(actual(codes.size()) == 1); wassert(actual(*codes.begin()) == WR_VAR(0, 12, 101)); }); add_method("multi", []() { set codes; resolve_varlist("B12101,B12103", codes); wassert(actual(codes.size()) == 2); set::const_iterator i = codes.begin(); wassert(actual(*i) == WR_VAR(0, 12, 101)); ++i; wassert(actual(*i) == WR_VAR(0, 12, 103)); }); add_method("format", []() { char buf[7]; format_code(WR_VAR(0, 0, 0), buf); wassert(actual(buf) == "B00000"); format_code(WR_VAR(0, 1, 2), buf); wassert(actual(buf) == "B01002"); format_code(WR_VAR(0, 63, 0), buf); wassert(actual(buf) == "B63000"); format_code(WR_VAR(0, 0, 255), buf); wassert(actual(buf) == "B00255"); format_code(WR_VAR(1, 1, 2), buf); wassert(actual(buf) == "R01002"); format_code(WR_VAR(2, 1, 2), buf); wassert(actual(buf) == "C01002"); format_code(WR_VAR(3, 1, 2), buf); wassert(actual(buf) == "D01002"); format_code(WR_VAR(4, 1, 2), buf); wassert(actual(buf) == "B01002"); format_bcode(WR_VAR(0, 1, 2), buf); wassert(actual(buf) == "B01002"); format_bcode(WR_VAR(1, 1, 2), buf); wassert(actual(buf) == "B01002"); format_bcode(WR_VAR(2, 1, 2), buf); wassert(actual(buf) == "B01002"); }); } } dballe-8.6/dballe/core/file-test.cc0000644000175000017500000000047713554564112014127 00000000000000#include "core/tests.h" #include "core/file.h" using namespace dballe; using namespace dballe::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("empty", []() { }); } } test("core_file"); } dballe-8.6/dballe/core/matcher.cc0000644000175000017500000001406013554564112013647 00000000000000#include "matcher.h" #include "defs.h" #include "query.h" #include #include #include using namespace std; using namespace wreport; namespace dballe { matcher::Result Matched::match_var_id(int) const { return matcher::MATCH_NA; } matcher::Result Matched::match_station_id(int) const { return matcher::MATCH_NA; } matcher::Result Matched::match_station_wmo(int, int) const { return matcher::MATCH_NA; } matcher::Result Matched::match_datetime(const DatetimeRange&) const { return matcher::MATCH_NA; } matcher::Result Matched::match_coords(const LatRange&, const LonRange&) const { return matcher::MATCH_NA; } matcher::Result Matched::match_rep_memo(const char* memo) const { return matcher::MATCH_NA; } matcher::Result Matched::int_in_range(int val, int min, int max) { if (min != MISSING_INT && val < min) return matcher::MATCH_NO; if (max != MISSING_INT && max < val) return matcher::MATCH_NO; return matcher::MATCH_YES; } matcher::Result Matched::lon_in_range(int val, int min, int max) { if (min == MISSING_INT && max == MISSING_INT) return matcher::MATCH_YES; if (min == MISSING_INT || max == MISSING_INT) throw error_consistency("both minimum and maximum values must be set when matching longitudes"); if (min < max) return (val >= min && val <= max) ? matcher::MATCH_YES : matcher::MATCH_NO; else return ((val >= min && val <= 18000000) || (val >= -18000000 && val <= max)) ? matcher::MATCH_YES : matcher::MATCH_NO; } namespace matcher { std::string result_format(Result res) { switch (res) { case MATCH_YES: return "yes"; case MATCH_NO: return "no"; case MATCH_NA: return "n/a"; } return "unknown"; } struct And : public Matcher { /** * Matchers added to exprs are *owned* and will be deallocated in the * destructor */ std::vector exprs; virtual ~And() { for (std::vector::iterator i = exprs.begin(); i != exprs.end(); ++i) delete *i; } Result match(const Matched& item) const { if (exprs.empty()) return MATCH_YES; Result res = MATCH_NA; for (std::vector::const_iterator i = exprs.begin(); i != exprs.end() && res != MATCH_NO; ++i) { switch ((*i)->match(item)) { case MATCH_YES: res = MATCH_YES; break; case MATCH_NO: res = MATCH_NO; break; case MATCH_NA: break; } } return res; } void to_query(core::Query& query) const override { for (std::vector::const_iterator i = exprs.begin(); i != exprs.end(); ++i) (*i)->to_query(query); } }; struct AnaIDMatcher : public Matcher { // Station ID to match int ana_id; AnaIDMatcher(int ana_id) : ana_id(ana_id) {} Result match(const Matched& v) const override { return v.match_station_id(ana_id) == MATCH_YES ? MATCH_YES : MATCH_NO; } void to_query(core::Query& query) const override { query.ana_id = ana_id; } }; struct WMOMatcher : public Matcher { int block; int station; WMOMatcher(int block, int station=-1) : block(block), station(station) {} Result match(const Matched& v) const override { return v.match_station_wmo(block, station) == MATCH_YES ? MATCH_YES : MATCH_NO; } void to_query(core::Query& query) const override { query.block = block; if (station != -1) query.station = station; else query.station = MISSING_INT; } }; struct DateMatcher : public Matcher { DatetimeRange range; DateMatcher(const DatetimeRange& range) : range(range) {} Result match(const Matched& v) const override { return v.match_datetime(range) == MATCH_YES ? MATCH_YES : MATCH_NO; } void to_query(core::Query& query) const override { query.dtrange = range; } }; struct CoordMatcher : public Matcher { LatRange latrange; LonRange lonrange; CoordMatcher(const LatRange& latrange, const LonRange& lonrange) : latrange(latrange), lonrange(lonrange) {} Result match(const Matched& v) const override { return v.match_coords(latrange, lonrange) == MATCH_YES ? MATCH_YES : MATCH_NO; } void to_query(core::Query& query) const override { query.latrange = latrange; query.lonrange = lonrange; } }; static string tolower(const std::string& s) { string res(s); for (string::iterator i = res.begin(); i != res.end(); ++i) *i = ::tolower(*i); return res; } struct ReteMatcher : public Matcher { string rete; ReteMatcher(const std::string& rete) : rete(tolower(rete)) {} Result match(const Matched& v) const override { return v.match_rep_memo(rete.c_str()) == MATCH_YES ? MATCH_YES : MATCH_NO; } void to_query(core::Query& query) const override { query.report = rete; } }; } std::unique_ptr Matcher::create(const dballe::Query& query_gen) { using namespace matcher; const core::Query& query = core::Query::downcast(query_gen); std::unique_ptr res(new And); if (query.ana_id != MISSING_INT) res->exprs.push_back(new AnaIDMatcher(query.ana_id)); if (query.block != MISSING_INT) { if (query.station != MISSING_INT) res->exprs.push_back(new WMOMatcher(query.block, query.station)); else res->exprs.push_back(new WMOMatcher(query.block)); } if (!query.dtrange.is_missing()) res->exprs.push_back(new DateMatcher(query.dtrange)); if (!query.latrange.is_missing() || !query.lonrange.is_missing()) res->exprs.push_back(new CoordMatcher(query.latrange, query.lonrange)); if (!query.report.empty()) res->exprs.push_back(new ReteMatcher(query.report)); return unique_ptr(res.release()); } } dballe-8.6/dballe/core/byteswap.h0000644000175000017500000000403313554564112013723 00000000000000#ifndef DBALLE_CORE_BYTESWAP_H #define DBALLE_CORE_BYTESWAP_H #include "config.h" #if USE_OWN_BSWAP /* byteswap.h - Byte swapping Copyright (C) 2005, 2007, 2009-2011 Free Software Foundation, Inc. Written by Oskar Liljeblad , 2005. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ /* Given an unsigned 16-bit argument X, return the value corresponding to X with reversed byte order. */ #define bswap_16(x) ((((x) & 0x00FF) << 8) | \ (((x) & 0xFF00) >> 8)) /* Given an unsigned 32-bit argument X, return the value corresponding to X with reversed byte order. */ #define bswap_32(x) ((((x) & 0x000000FF) << 24) | \ (((x) & 0x0000FF00) << 8) | \ (((x) & 0x00FF0000) >> 8) | \ (((x) & 0xFF000000) >> 24)) /* Given an unsigned 64-bit argument X, return the value corresponding to X with reversed byte order. */ #define bswap_64(x) ((((x) & 0x00000000000000FFULL) << 56) | \ (((x) & 0x000000000000FF00ULL) << 40) | \ (((x) & 0x0000000000FF0000ULL) << 24) | \ (((x) & 0x00000000FF000000ULL) << 8) | \ (((x) & 0x000000FF00000000ULL) >> 8) | \ (((x) & 0x0000FF0000000000ULL) >> 24) | \ (((x) & 0x00FF000000000000ULL) >> 40) | \ (((x) & 0xFF00000000000000ULL) >> 56)) #else #include #endif #endif dballe-8.6/dballe/core/trace.h0000644000175000017500000000106513554564112013165 00000000000000/** @file * Debugging tracing functions */ #ifndef DBALLE_CORE_TRACE_H #define DBALLE_CORE_TRACE_H /* * Include this file if you want to enable trace functions in a source * * The trace functions are not compiled unless you #define TRACE_SOURCE * before including this header. */ #ifdef TRACE_SOURCE #include // Output a trace message #define TRACE(...) fprintf(stderr, __VA_ARGS__) // Prefix a block of code to compile only if trace is enabled #define IFTRACE if (1) #else #define TRACE(...) do { } while (0) #define IFTRACE if (0) #endif #endif dballe-8.6/dballe/core/match-wreport.h0000644000175000017500000000364513554564112014671 00000000000000#ifndef DBALLE_CORE_MATCH_WREPORT_H #define DBALLE_CORE_MATCH_WREPORT_H /** @file * @ingroup core * Implement a storage object for a group of related observation data */ #include namespace wreport { struct Var; struct Subset; struct Bulletin; } namespace dballe { struct MatchedSubset : public Matched { const wreport::Subset& r; MatchedSubset(const wreport::Subset& r); ~MatchedSubset(); /** * Return YES if the subset contains at least one var with the given B33195 * attribute; else return NA. */ matcher::Result match_var_id(int val) const override; matcher::Result match_station_id(int val) const override; matcher::Result match_station_wmo(int block, int station=-1) const override; matcher::Result match_datetime(const DatetimeRange& range) const override; matcher::Result match_coords(const LatRange& latrange, const LonRange& lonrange) const override; matcher::Result match_rep_memo(const char* memo) const override; protected: Datetime date; int lat, lon; const wreport::Var* var_ana_id; const wreport::Var* var_block; const wreport::Var* var_station; const wreport::Var* var_rep_memo; }; /** * Match all subsets in turn, returning true if at least one subset matches */ struct MatchedBulletin : public Matched { const wreport::Bulletin& r; MatchedBulletin(const wreport::Bulletin& r); ~MatchedBulletin(); matcher::Result match_var_id(int val) const override; matcher::Result match_station_id(int val) const override; matcher::Result match_station_wmo(int block, int station=-1) const override; matcher::Result match_datetime(const DatetimeRange& range) const override; matcher::Result match_coords(const LatRange& latrange, const LonRange& lonrange) const override; matcher::Result match_rep_memo(const char* memo) const override; protected: const MatchedSubset** subsets; }; } #endif dballe-8.6/dballe/core/benchmark.cc0000644000175000017500000001454013554564112014161 00000000000000#include "benchmark.h" #include #include #include #include #include #include #include "dballe/msg/msg.h" #include "dballe/importer.h" using namespace std; /* namespace { double ticks_per_sec = sysconf(_SC_CLK_TCK); } */ namespace dballe { namespace benchmark { /* void Task::collect(std::function f) { run_count += 1; struct tms tms_start, tms_end; times(&tms_start); f(); times(&tms_end); utime += tms_end.tms_utime - tms_start.tms_utime; stime += tms_end.tms_stime - tms_start.tms_stime; } */ static void bench_getrusage(int who, struct rusage *usage) { if (::getrusage(RUSAGE_SELF, usage) == -1) throw std::system_error(errno, std::system_category(), "getrusage failed"); } static void bench_clock_gettime(clockid_t clk_id, struct timespec *res) { if (::clock_gettime(clk_id, res) == -1) throw std::system_error(errno, std::system_category(), "clock_gettime failed"); } std::string format_clockdiff(const struct timespec& begin, const struct timespec& until) { unsigned long secs = 0; unsigned long nsecs = 0; if (begin.tv_nsec <= until.tv_nsec) { secs = until.tv_sec - begin.tv_sec; nsecs = until.tv_nsec - begin.tv_nsec; } else { secs = until.tv_sec - begin.tv_sec - 1; nsecs = 1000000000 + until.tv_nsec - begin.tv_nsec; } char buf[32]; if (secs > 0) snprintf(buf, 32, "%lu.%03lus", secs, nsecs / 1000000); else if (nsecs > 1000000) snprintf(buf, 32, "%lums", nsecs / 1000000); else if (nsecs > 1000) snprintf(buf, 32, "%luµs", nsecs / 1000); else snprintf(buf, 32, "%luns", nsecs); return buf; } void Timeit::run(Progress& progress, Task& task) { task_name = task.name(); progress.start_timeit(*this); try { task.setup(); bench_getrusage(RUSAGE_SELF, &res_at_start); bench_clock_gettime(CLOCK_MONOTONIC_RAW, &time_at_start); for (unsigned i = 0; i < repetitions; ++i) task.run_once(); bench_clock_gettime(CLOCK_MONOTONIC_RAW, &time_at_end); bench_getrusage(RUSAGE_SELF, &res_at_end); } catch (std::exception& e) { progress.test_failed(task, e); } task.teardown(); progress.end_timeit(*this); } void Throughput::run(Progress& progress, Task& task) { task_name = task.name(); progress.start_throughput(*this); try { task.setup(); struct timespec time_at_start; bench_clock_gettime(CLOCK_MONOTONIC_RAW, &time_at_start); struct timespec time_at_end; time_at_end.tv_nsec = time_at_start.tv_nsec + ((long)floor(run_time * 1000000000.0) % 1000000000); time_at_end.tv_sec = time_at_start.tv_sec + time_at_end.tv_nsec / 1000000000 + (long)floor(run_time); time_at_end.tv_nsec = time_at_end.tv_nsec % 1000000000; struct timespec time_cur; for ( ; true; ++times_run) { bench_clock_gettime(CLOCK_MONOTONIC_RAW, &time_cur); if (time_cur.tv_sec > time_at_end.tv_sec) break; if (time_cur.tv_sec == time_at_end.tv_sec && time_cur.tv_nsec > time_at_end.tv_nsec) break; task.run_once(); } run_time = time_cur.tv_sec - time_at_start.tv_sec + (time_cur.tv_nsec - time_at_start.tv_nsec) / 1000000000.0; } catch (std::exception& e) { progress.test_failed(task, e); } task.teardown(); progress.end_throughput(*this); } Benchmark::Benchmark() : progress(make_shared()) { } Benchmark::~Benchmark() {} void Benchmark::timeit(Task& task, unsigned repetitions) { timeit_tasks.emplace_back(Timeit()); timeit_tasks.back().repetitions = repetitions; timeit_tasks.back().run(*progress, task); } void Benchmark::throughput(Task& task, double run_time) { throughput_tasks.emplace_back(Throughput()); throughput_tasks.back().run_time = run_time; throughput_tasks.back().run(*progress, task); } void Benchmark::print_timings() { for (auto& t: timeit_tasks) { string time = format_clockdiff(t.time_at_start, t.time_at_end); fprintf(stdout, "%s,%u,%s\n", t.task_name.c_str(), t.repetitions, time.c_str()); } for (auto& t: throughput_tasks) { fprintf(stdout, "%s,%.2f,%d\n", t.task_name.c_str(), t.run_time, t.times_run); } /* for (auto& t: tasks) { fprintf(stdout, "%s.%s: %d runs, user: %.2fs (%.1f%%), sys: %.2fs (%.1f%%), total: %.2fs (%.1f%%)\n", name.c_str(), t->name.c_str(), t->run_count, t->utime / ticks_per_sec, t->utime * 100.0 / task_main.utime, t->stime / ticks_per_sec, t->stime * 100.0 / task_main.stime, (t->utime + t->stime) / ticks_per_sec, (t->utime + t->stime) * 100.0 / (task_main.utime + task_main.stime)); } */ } BasicProgress::BasicProgress(FILE* out, FILE* err) : out(out), err(err) {} void BasicProgress::start_timeit(const Timeit& t) { fprintf(out, "%s: starting...\n", t.task_name.c_str()); } void BasicProgress::end_timeit(const Timeit& t) { fprintf(out, "%s: done.\n", t.task_name.c_str()); } void BasicProgress::start_throughput(const Throughput& t) { fprintf(out, "%s: ", t.task_name.c_str()); fflush(out); } void BasicProgress::end_throughput(const Throughput& t) { fprintf(out, "%u times in %.2fs: %.2f/s.\n", t.times_run, t.run_time, (double)t.times_run/t.run_time); } void BasicProgress::test_failed(const Task& t, std::exception& e) { fprintf(err, "%s: failed: %s\n", t.name(), e.what()); } void Messages::load(const std::string& pathname, dballe::Encoding encoding, const char* codec_options) { auto importer = Importer::create(Encoding::BUFR, codec_options); auto in = File::create(encoding, pathname, "rb"); in->foreach([&](const BinaryMessage& rmsg) { emplace_back(importer->from_binary(rmsg)); return true; }); } void Messages::duplicate(size_t size, const Datetime& datetime) { for (size_t i = 0; i < size; ++i) emplace_back((*this)[i]); } Whitelist::Whitelist(int argc, const char* argv[]) { for (int i = 1; i < argc; ++i) emplace_back(argv[i]); } bool Whitelist::has(const std::string& val) { if (empty()) return true; return std::find(begin(), end(), val) != end(); } } } dballe-8.6/dballe/core/query-access.in.cc0000644000175000017500000001465513572423710015245 00000000000000#include "query.h" #include "var.h" #include using namespace wreport; namespace dballe { namespace core { void Query::setf(const char* key, unsigned len, const char* val) { switch (key) { // mklookup case "priority": priomin = priomax = strtol(val, nullptr, 10); case "priomax": priomax = strtol(val, nullptr, 10); case "priomin": priomin = strtol(val, nullptr, 10); case "rep_memo": report = val; case "report": report = val; case "ana_id": ana_id = strtol(val, nullptr, 10); case "mobile": mobile = strtol(val, nullptr, 10); case "ident": ident = val; case "lat": { double dval = strtod(val, nullptr); latrange.set(dval, dval); } case "lon": { double dval = strtod(val, nullptr); lonrange.set(dval, dval); } case "latmax": latrange.imax = Coords::lat_to_int(strtod(val, nullptr)); case "latmin": latrange.imin = Coords::lat_to_int(strtod(val, nullptr)); case "lonmax": lonrange.imax = Coords::lon_to_int(strtod(val, nullptr)); case "lonmin": lonrange.imin = Coords::lon_to_int(strtod(val, nullptr)); case "year": dtrange.min.year = dtrange.max.year = strtol(val, nullptr, 10); case "month": dtrange.min.month = dtrange.max.month = strtol(val, nullptr, 10); case "day": dtrange.min.day = dtrange.max.day = strtol(val, nullptr, 10); case "hour": dtrange.min.hour = dtrange.max.hour = strtol(val, nullptr, 10); case "min": dtrange.min.minute = dtrange.max.minute = strtol(val, nullptr, 10); case "sec": dtrange.min.second = dtrange.max.second = strtol(val, nullptr, 10); case "yearmax": dtrange.max.year = strtol(val, nullptr, 10); case "yearmin": dtrange.min.year = strtol(val, nullptr, 10); case "monthmax": dtrange.max.month = strtol(val, nullptr, 10); case "monthmin": dtrange.min.month = strtol(val, nullptr, 10); case "daymax": dtrange.max.day = strtol(val, nullptr, 10); case "daymin": dtrange.min.day = strtol(val, nullptr, 10); case "hourmax": dtrange.max.hour = strtol(val, nullptr, 10); case "hourmin": dtrange.min.hour = strtol(val, nullptr, 10); case "minumax": dtrange.max.minute = strtol(val, nullptr, 10); case "minumin": dtrange.min.minute = strtol(val, nullptr, 10); case "secmax": dtrange.max.second = strtol(val, nullptr, 10); case "secmin": dtrange.min.second = strtol(val, nullptr, 10); case "leveltype1": level.ltype1 = strtol(val, nullptr, 10); case "l1": level.l1 = strtol(val, nullptr, 10); case "leveltype2": level.ltype2 = strtol(val, nullptr, 10); case "l2": level.l2 = strtol(val, nullptr, 10); case "pindicator": trange.pind = strtol(val, nullptr, 10); case "p1": trange.p1 = strtol(val, nullptr, 10); case "p2": trange.p2 = strtol(val, nullptr, 10); case "var": varcodes.clear(); varcodes.insert(resolve_varcode(val)); case "varlist": varcodes.clear(); resolve_varlist(val, varcodes); case "query": query = val; case "ana_filter": ana_filter = val; case "data_filter": data_filter = val; case "attr_filter": attr_filter = val; case "limit": limit = strtol(val, nullptr, 10); case "block": block = strtol(val, nullptr, 10); case "station": station = strtol(val, nullptr, 10); default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } void Query::unset(const char* key, unsigned len) { switch (key) { // mklookup case "priority": priomin = priomax = MISSING_INT; case "priomax": priomax = MISSING_INT; case "priomin": priomin = MISSING_INT; case "rep_memo": report.clear(); case "report": report.clear(); case "ana_id": ana_id = MISSING_INT; case "mobile": mobile = MISSING_INT; case "ident": ident.clear(); case "lat": latrange = LatRange(); case "lon": lonrange = LonRange(); case "latmax": latrange.imax = LatRange::IMAX; case "latmin": latrange.imin = LatRange::IMIN; case "lonmax": lonrange.imax = MISSING_INT; case "lonmin": lonrange.imin = MISSING_INT; case "year": dtrange.min.year = dtrange.max.year = 0xffff; case "month": dtrange.min.month = dtrange.max.month = 0xff; case "day": dtrange.min.day = dtrange.max.day = 0xff; case "hour": dtrange.min.hour = dtrange.max.hour = 0xff; case "min": dtrange.min.minute = dtrange.max.minute = 0xff; case "sec": dtrange.min.second = dtrange.max.second = 0xff; case "yearmax": dtrange.max.year = 0xffff; case "yearmin": dtrange.min.year = 0xffff; case "monthmax": dtrange.max.month = 0xff; case "monthmin": dtrange.min.month = 0xff; case "daymax": dtrange.max.day = 0xff; case "daymin": dtrange.min.day = 0xff; case "hourmax": dtrange.max.hour = 0xff; case "hourmin": dtrange.min.hour = 0xff; case "minumax": dtrange.max.minute = 0xff; case "minumin": dtrange.min.minute = 0xff; case "secmax": dtrange.max.second = 0xff; case "secmin": dtrange.min.second = 0xff; case "leveltype1": level.ltype1 = MISSING_INT; case "l1": level.l1 = MISSING_INT; case "leveltype2": level.ltype2 = MISSING_INT; case "l2": level.l2 = MISSING_INT; case "pindicator": trange.pind = MISSING_INT; case "p1": trange.p1 = MISSING_INT; case "p2": trange.p2 = MISSING_INT; case "var": varcodes.clear(); case "varlist": varcodes.clear(); case "query": query.clear(); case "ana_filter": ana_filter.clear(); case "data_filter": data_filter.clear(); case "attr_filter": attr_filter.clear(); case "limit": limit = MISSING_INT; case "block": block = MISSING_INT; case "station": station = MISSING_INT; default: wreport::error_notfound::throwf("key %s is not valid for a query", key); } } } } dballe-8.6/dballe/db/0000755000175000017500000000000013602152021011413 500000000000000dballe-8.6/dballe/db/db-query-summary-test.cc0000644000175000017500000003065313554564112016067 00000000000000#include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct DBData : public TestDataSet { DBData() { stations["st1_synop"].station.coords = Coords(12.34560, 76.54320); stations["st1_synop"].station.report = "synop"; stations["st1_synop"].values.set(newvar("B07030", 42.0)); // height stations["st1_metar"].station = stations["st1_synop"].station; stations["st1_metar"].station.report = "metar"; stations["st1_metar"].values.set(newvar("block", 1)); stations["st1_metar"].values.set(newvar("station", 2)); stations["st1_metar"].values.set(newvar("B07030", 50.0)); // height stations["st2_temp"].station.coords = Coords(23.45670, 65.43210); stations["st2_temp"].station.report = "temp"; stations["st2_temp"].values.set(newvar("B07030", 100.0)); // height stations["st2_metar"].station = stations["st2_temp"].station; stations["st2_metar"].station.report = "metar"; stations["st2_metar"].values.set(newvar("block", 3)); stations["st2_metar"].values.set(newvar("station", 4)); stations["st2_metar"].values.set(newvar("B07030", 110.0)); // height data["rec1a"].station = stations["st1_metar"].station; data["rec1a"].datetime = Datetime(1945, 4, 25, 8); data["rec1a"].level = Level(10, 11, 15, 22); data["rec1a"].trange = Trange(20, 111, 122); data["rec1a"].values.set("B12101", 290.0); data["rec1a"].values.set("B12103", 280.0); data["rec1b"] = data["rec1a"]; data["rec1b"].datetime = Datetime(1945, 4, 26, 8); data["rec1b"].values.set("B12101", 291.0); data["rec1b"].values.set("B12103", 281.0); data["rec2a"].station = stations["st2_metar"].station; data["rec2a"].datetime = Datetime(1945, 4, 25, 8); data["rec2a"].level = Level(10, 11, 15, 22); data["rec2a"].trange = Trange(20, 111, 122); data["rec2a"].values.set("B12101", 300.0); data["rec2a"].values.set("B12103", 298.0); data["rec2b"] = data["rec2a"]; data["rec2b"].datetime = Datetime(1945, 4, 26, 8); data["rec2b"].values.set("B12101", 301.0); data["rec2b"].values.set("B12103", 291.0); } }; template struct DBDataFixture : public TransactionFixture { using TransactionFixture::TransactionFixture; int st1_id; int st2_id; void create_db() override { TransactionFixture::create_db(); st1_id = this->test_data.stations["st1_metar"].station.id; st2_id = this->test_data.stations["st2_metar"].station.id; } }; std::string parm(const char* name, int val) { stringstream out; out << name << "=" << val; return out.str(); } template class Tests : public FixtureTestCase> { typedef DBDataFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override { this->add_method("query_ana_id", [](Fixture& f) { wassert(actual(f.tr).try_summary_query(parm("ana_id", f.st1_id), 2)); wassert(actual(f.tr).try_summary_query(parm("ana_id", f.st2_id), 2)); wassert(actual(f.tr).try_summary_query(parm("ana_id", (f.st1_id + f.st2_id) * 2), 0)); }); #if 0 // TODO: summary of station vars is not supported at the moment, waiting for a use case for it this->add_method("query_station_vars", [](Fixture& f) { auto& db = *f.db; core::Query query; query.query_station_vars = true; auto cur = db.query_summary(query); ensure_equals(cur->test_iterate(), 8); }); #endif this->add_method("query_year", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("year=1001", 0)); wassert(actual(f.tr).try_summary_query("yearmin=1999", 0)); auto check_base = [](const db::DBSummary& res) { wassert(actual(res.stations().size()) == 2u); wassert(actual(res.stations().begin()->station.coords) == Coords(12.34560, 76.54320)); wassert(actual(res.levels().size()) == 1u); wassert(actual(*res.levels().begin()) == Level(10, 11, 15, 22)); wassert(actual(res.tranges().size()) == 1u); wassert(actual(*res.tranges().begin()) == Trange(20, 111, 122)); wassert(actual(res.varcodes().size()) == 2u); wassert(actual(*res.varcodes().begin()) == WR_VAR(0, 12, 101)); wassert(actual(*(res.varcodes().begin() + 1)) == WR_VAR(0, 12, 103)); }; auto check_nodetails = [&](const db::DBSummary& res) { wassert(check_base(res)); wassert(actual(res.data_count()) == 0); wassert_true(res.datetime_min().is_missing()); wassert_true(res.datetime_max().is_missing()); }; auto check_details = [&](const db::DBSummary& res) { wassert(check_base(res)); wassert(actual(res.data_count()) == 8u); wassert(actual(res.datetime_min()) == Datetime(1945, 4, 25, 8)); wassert(actual(res.datetime_max()) == Datetime(1945, 4, 26, 8)); auto entry = res.stations().begin(); auto varentry = entry->begin(); wassert(actual(varentry->count) == 2u); wassert(actual(varentry->dtrange.min) == Datetime(1945, 4, 25, 8)); wassert(actual(varentry->dtrange.max) == Datetime(1945, 4, 26, 8)); }; wassert(actual(f.tr).try_summary_query("yearmin=1945", 4, check_nodetails)); wassert(actual(f.tr).try_summary_query("yearmin=1945, query=details", 4, check_details)); wassert(actual(f.tr).try_summary_query("yearmax=1944", 0)); wassert(actual(f.tr).try_summary_query("yearmax=1945", 4)); wassert(actual(f.tr).try_summary_query("yearmax=2030", 4)); wassert(actual(f.tr).try_summary_query("year=1944", 0)); wassert(actual(f.tr).try_summary_query("year=1945", 4)); wassert(actual(f.tr).try_summary_query("year=1946", 0)); }); this->add_method("query_blockstation", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("block=1", 2)); wassert(actual(f.tr).try_summary_query("block=2", 0)); wassert(actual(f.tr).try_summary_query("station=3", 0)); wassert(actual(f.tr).try_summary_query("station=4", 2)); }); this->add_method("query_ana_filter", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("ana_filter=block=1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=B01001=1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=block>1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=B01001>1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=block<=1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=B01001>3", 0)); wassert(actual(f.tr).try_summary_query("ana_filter=B01001>=3", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=B01001<=1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=0<=B01001<=2", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=1<=B01001<=1", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=2<=B01001<=4", 2)); wassert(actual(f.tr).try_summary_query("ana_filter=4<=B01001<=6", 0)); }); this->add_method("query_data_filter", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("data_filter=B12101<300.0", 1)); wassert(actual(f.tr).try_summary_query("data_filter=B12101<=300.0", 2)); wassert(actual(f.tr).try_summary_query("data_filter=B12101=300.0", 1)); wassert(actual(f.tr).try_summary_query("data_filter=B12101>=300,0", 1)); wassert(actual(f.tr).try_summary_query("data_filter=B12101>300.0", 1)); wassert(actual(f.tr).try_summary_query("data_filter=B12101<400.0", 2)); wassert(actual(f.tr).try_summary_query("data_filter=B12101<=400.0", 2)); wassert(actual(f.tr).try_summary_query("data_filter=B12102>400.0", 0)); }); this->add_method("query_lat_lon", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("latmin=11.0", 4)); wassert(actual(f.tr).try_summary_query("latmin=12.34560", 4)); wassert(actual(f.tr).try_summary_query("latmin=13.0", 2)); wassert(actual(f.tr).try_summary_query("latmax=11.0", 0)); wassert(actual(f.tr).try_summary_query("latmax=12.34560", 2)); wassert(actual(f.tr).try_summary_query("latmax=13.0", 2)); wassert(actual(f.tr).try_summary_query("lonmin=75., lonmax=77.", 2)); wassert(actual(f.tr).try_summary_query("lonmin=76.54320, lonmax=76.54320", 2)); wassert(actual(f.tr).try_summary_query("lonmin=76.54330, lonmax=77.", 0)); wassert(actual(f.tr).try_summary_query("lonmin=77., lonmax=76.54310", 2)); wassert(actual(f.tr).try_summary_query("lonmin=77., lonmax=76.54320", 4)); wassert(actual(f.tr).try_summary_query("lonmin=77., lonmax=-10", 0)); }); this->add_method("query_mobile", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("mobile=0", 4)); wassert(actual(f.tr).try_summary_query("mobile=1", 0)); }); this->add_method("query_ident", [](Fixture& f) { //auto& db = *f.db; // TODO: add mobile stations to the fixture so we can query ident }); this->add_method("query_timerange", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("pindicator=20", 4)); wassert(actual(f.tr).try_summary_query("pindicator=21", 0)); wassert(actual(f.tr).try_summary_query("p1=111", 4)); wassert(actual(f.tr).try_summary_query("p1=112", 0)); wassert(actual(f.tr).try_summary_query("p2=121", 0)); wassert(actual(f.tr).try_summary_query("p2=122", 4)); wassert(actual(f.tr).try_summary_query("p2=123", 0)); }); this->add_method("query_level", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("leveltype1=10", 4)); wassert(actual(f.tr).try_summary_query("leveltype1=11", 0)); wassert(actual(f.tr).try_summary_query("leveltype2=15", 4)); wassert(actual(f.tr).try_summary_query("leveltype2=16", 0)); wassert(actual(f.tr).try_summary_query("l1=11", 4)); wassert(actual(f.tr).try_summary_query("l1=12", 0)); wassert(actual(f.tr).try_summary_query("l2=22", 4)); wassert(actual(f.tr).try_summary_query("l2=23", 0)); }); this->add_method("query_var", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("var=B01001", 0)); wassert(actual(f.tr).try_summary_query("var=B12101", 2)); wassert(actual(f.tr).try_summary_query("var=B12102", 0)); }); this->add_method("query_rep_memo", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("rep_memo=synop", 0)); wassert(actual(f.tr).try_summary_query("rep_memo=metar", 4)); wassert(actual(f.tr).try_summary_query("rep_memo=temp", 0)); }); this->add_method("query_priority", [](Fixture& f) { wassert(actual(f.tr).try_summary_query("priority=101", 0)); wassert(actual(f.tr).try_summary_query("priority=81", 4)); wassert(actual(f.tr).try_summary_query("priority=102", 0)); wassert(actual(f.tr).try_summary_query("priomin=70", 4)); wassert(actual(f.tr).try_summary_query("priomin=80", 4)); wassert(actual(f.tr).try_summary_query("priomin=90", 0)); wassert(actual(f.tr).try_summary_query("priomax=70", 0)); wassert(actual(f.tr).try_summary_query("priomax=81", 4)); wassert(actual(f.tr).try_summary_query("priomax=100", 4)); }); } }; Tests tg2("db_query_summary_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_query_summary_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_query_summary_v7_mysql", "MYSQL"); #endif } dballe-8.6/dballe/db/db-query-station-test.cc0000644000175000017500000002311613554564112016047 00000000000000#include "config.h" #include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/station.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct DBData : public TestDataSet { DBData() { stations["st1_synop"].station.coords = Coords(12.34560, 76.54320); stations["st1_synop"].station.report = "synop"; stations["st1_synop"].values.set(newvar("block", 1)); stations["st1_synop"].values.set(newvar("station", 1)); stations["st1_synop"].values.set(newvar("B07030", 42.0)); // height stations["st1_metar"].station = stations["st1_synop"].station; stations["st1_metar"].station.report = "metar"; stations["st1_metar"].values.set(newvar("block", 1)); stations["st1_metar"].values.set(newvar("station", 2)); stations["st1_metar"].values.set(newvar("B07030", 50.0)); // height stations["st2_temp"].station.coords = Coords(23.45670, 65.43210); stations["st2_temp"].station.report = "temp"; stations["st2_temp"].values.set(newvar("block", 3)); stations["st2_temp"].values.set(newvar("station", 4)); stations["st2_temp"].values.set(newvar("B07030", 100.0)); // height stations["st2_metar"].station = stations["st2_temp"].station; stations["st2_metar"].station.report = "metar"; stations["st2_metar"].values.set(newvar("block", 3)); stations["st2_metar"].values.set(newvar("station", 4)); stations["st2_metar"].values.set(newvar("B07030", 110.0)); // height data["rec1"].station = stations["st1_metar"].station; data["rec1"].datetime = Datetime(1945, 4, 25, 8); data["rec1"].level = Level(10, 11, 15, 22); data["rec1"].trange = Trange(20, 111, 122); data["rec1"].values.set("B12101", 290.0); data["rec2"].station = stations["st2_metar"].station; data["rec2"].datetime = Datetime(1945, 4, 25, 8); data["rec2"].level = Level(10, 11, 15, 22); data["rec2"].trange = Trange(20, 111, 122); data["rec2"].values.set("B12101", 300.0); data["rec2"].values.set("B12103", 298.0); } }; template class Tests : public FixtureTestCase> { typedef TransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override { this->add_method("query_ana_id", [](Fixture& f) { wassert(actual(f.tr).try_station_query("ana_id=1", 1)); wassert(actual(f.tr).try_station_query("ana_id=2", 1)); }); this->add_method("query_lat_lon", [](Fixture& f) { wassert(actual(f.tr).try_station_query("lat=12.00000", 0)); wassert(actual(f.tr).try_station_query("lat=12.34560", 2)); wassert(actual(f.tr).try_station_query("lat=23.45670", 2)); wassert(actual(f.tr).try_station_query("latmin=12.00000", 4)); wassert(actual(f.tr).try_station_query("latmin=12.34560", 4)); wassert(actual(f.tr).try_station_query("latmin=12.34570", 2)); wassert(actual(f.tr).try_station_query("latmin=23.45670", 2)); wassert(actual(f.tr).try_station_query("latmin=23.45680", 0)); wassert(actual(f.tr).try_station_query("latmax=12.00000", 0)); wassert(actual(f.tr).try_station_query("latmax=12.34560", 2)); wassert(actual(f.tr).try_station_query("latmax=12.34570", 2)); wassert(actual(f.tr).try_station_query("latmax=23.45670", 4)); wassert(actual(f.tr).try_station_query("latmax=23.45680", 4)); wassert(actual(f.tr).try_station_query("lon=76.00000", 0)); wassert(actual(f.tr).try_station_query("lon=76.54320", 2)); wassert(actual(f.tr).try_station_query("lon=65.43210", 2)); wassert(actual(f.tr).try_station_query("lonmin=10., lonmax=20.", 0)); wassert(actual(f.tr).try_station_query("lonmin=76.54320, lonmax=76.54320", 2)); wassert(actual(f.tr).try_station_query("lonmin=76.54320, lonmax=77.", 2)); wassert(actual(f.tr).try_station_query("lonmin=76.54330, lonmax=77.", 0)); wassert(actual(f.tr).try_station_query("lonmin=60., lonmax=77.", 4)); wassert(actual(f.tr).try_station_query("lonmin=77., lonmax=76.54310", 2)); wassert(actual(f.tr).try_station_query("lonmin=77., lonmax=76.54320", 4)); wassert(actual(f.tr).try_station_query("lonmin=77., lonmax=-10", 0)); }); this->add_method("query_mobile", [](Fixture& f) { wassert(actual(f.tr).try_station_query("mobile=0", 4)); wassert(actual(f.tr).try_station_query("mobile=1", 0)); }); this->add_method("query_ident", [](Fixture& f) { // FIXME: add some mobile stations to the test fixture to test ident }); this->add_method("query_block_station", [](Fixture& f) { wassert(actual(f.tr).try_station_query("block=1", 2)); wassert(actual(f.tr).try_station_query("block=2", 0)); wassert(actual(f.tr).try_station_query("block=3", 2)); wassert(actual(f.tr).try_station_query("block=4", 0)); wassert(actual(f.tr).try_station_query("station=1", 1)); wassert(actual(f.tr).try_station_query("station=2", 1)); wassert(actual(f.tr).try_station_query("station=3", 0)); wassert(actual(f.tr).try_station_query("station=4", 2)); }); this->add_method("query_mobile", [](Fixture& f) { }); this->add_method("query_ana_filter", [](Fixture& f) { wassert(actual(f.tr).try_station_query("ana_filter=block=1", 2)); wassert(actual(f.tr).try_station_query("ana_filter=block=2", 0)); wassert(actual(f.tr).try_station_query("ana_filter=block=3", 2)); wassert(actual(f.tr).try_station_query("ana_filter=block>=1", 4)); wassert(actual(f.tr).try_station_query("ana_filter=B07030=42", 1)); wassert(actual(f.tr).try_station_query("ana_filter=B07030=50", 1)); wassert(actual(f.tr).try_station_query("ana_filter=B07030=100", 1)); wassert(actual(f.tr).try_station_query("ana_filter=B07030=110", 1)); wassert(actual(f.tr).try_station_query("ana_filter=B07030=120", 0)); wassert(actual(f.tr).try_station_query("ana_filter=B07030>50", 2)); wassert(actual(f.tr).try_station_query("ana_filter=B07030>=50", 3)); wassert(actual(f.tr).try_station_query("ana_filter=50<=B07030<=100", 2)); }); this->add_method("query_var", [](Fixture& f) { /* * Querying var= or varlist= on a station query means querying stations * that measure that variable or those variables. */ wassert(actual(f.tr).try_station_query("var=B12101", 2)); wassert(actual(f.tr).try_station_query("var=B12103", 1)); wassert(actual(f.tr).try_station_query("varlist=B12101", 2)); wassert(actual(f.tr).try_station_query("varlist=B12103", 1)); wassert(actual(f.tr).try_station_query("varlist=B12101,B12103", 2)); }); this->add_method("stations_without_data", [](Fixture& f) { // Manually insert an orphan station switch (DB::format) { case Format::V7: if (auto t = dynamic_cast(f.tr.get())) { v7::Tracer<> trc; dballe::DBStation station; station.report = "synop"; station.coords = Coords(1100000, 4500000); station.ident = "ciao"; wassert(t->station().insert_new(trc, station)); } break; default: error_unimplemented::throwf("cannot run this test on a database of format %d", (int)DB::format); } // Query stations and make sure that they do not appear. They should // not appear, but they currently do because of a bug. I need to // preserve the bug until the software that relies on it has been // migrated to use standard DB-All.e features. core::Query query; query.latrange.set(11.0, 11.0); query.lonrange.set(45.0, 45.0); auto cur = f.tr->query_stations(query); #warning TODO: fix this test to give an error once we do not need to support this bug anymore //wassert(actual(cur->remaining()) == 0); wassert(actual(cur->remaining()) == 1); }); this->add_method("query_ordering", [](Fixture& f) { auto cur = f.tr->query_stations(core::Query()); switch (f.db->format()) { case Format::V7: wassert(actual(cur->remaining()) == 4); break; default: error_unimplemented::throwf("cannot run this test on a database of format %d", (int)DB::format); } }); this->add_method("query_rep_memo", [](Fixture& f) { // https://github.com/ARPA-SIMC/dballe/issues/35 wassert(actual(f.tr).try_station_query("rep_memo=synop", 1)); wassert(actual(f.tr).try_station_query("rep_memo=metar", 2)); }); } }; Tests tg2("db_query_station_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_query_station_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_query_station_v7_mysql", "MYSQL"); #endif } dballe-8.6/dballe/db/db-import-test.cc0000644000175000017500000003365513554573614014555 00000000000000#include "config.h" #include "dballe/db/tests.h" #include "v7/db.h" #include "v7/transaction.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include #include using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { static impl::DBImportOptions default_opts; unsigned diff_msg(std::shared_ptr first, std::shared_ptr second, const char* tag) { notes::Collect c(cerr); int diffs = first->diff(*second); if (diffs) dballe::tests::track_different_msgs(*first, *second, tag); return diffs; } template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override { default_opts.import_attributes = true; default_opts.update_station = true; this->add_method("crex", [](Fixture& f) { core::Query query; // Test import/export with all CREX samples const char** files = dballe::tests::crex_files; set blacklist; // These files have no data to import blacklist.insert("crex/test-synop1.crex"); blacklist.insert("crex/test-synop3.crex"); for (int i = 0; files[i] != NULL; ++i) { if (blacklist.find(files[i]) != blacklist.end()) continue; try { impl::Messages inmsgs = read_msgs(files[i], Encoding::CREX); auto msg = impl::Message::downcast(inmsgs[0]); f.tr->remove_all(); f.tr->import_message(*msg, default_opts); // Explicitly set the rep_memo variable that is added during export msg->set_rep_memo(impl::Message::repmemo_from_type(msg->get_type())); query.clear(); query.report = impl::Message::repmemo_from_type(msg->get_type()); impl::Messages msgs = wcallchecked(dballe::tests::messages_from_db(f.tr, query)); wassert(actual(msgs.size()) == 1u); wassert(actual(diff_msg(msg, msgs[0], "crex")) == 0); } catch (std::exception& e) { wassert(throw TestFailed(string("[") + files[i] + "] " + e.what())); } } }); this->add_method("bufr", [](Fixture& f) { // Test import/export with all BUFR samples core::Query query; const char** files = dballe::tests::bufr_files; for (int i = 0; files[i] != NULL; i++) { try { impl::Messages inmsgs = read_msgs(files[i], Encoding::BUFR); auto msg = impl::Message::downcast(inmsgs[0]); f.tr->remove_all(); wassert(f.tr->import_message(*msg, default_opts)); query.clear(); query.report = impl::Message::repmemo_from_type(msg->type); impl::Messages msgs = dballe::tests::messages_from_db(f.tr, query); wassert(actual(msgs.size()) == 1u); // Explicitly set the rep_memo variable that is added during export msg->set_rep_memo(impl::Message::repmemo_from_type(msg->type)); wassert(actual(diff_msg(msg, msgs[0], "bufr")) == 0); } catch (std::exception& e) { wassert(throw TestFailed(string("[") + files[i] + "] " + e.what())); } } }); this->add_method("multi", [](Fixture& f) { // Check that multiple messages are correctly identified during export core::Query query; // msg1 has latitude 33.88 // msg2 has latitude 46.22 impl::Messages msgs1 = read_msgs("bufr/obs0-1.22.bufr", Encoding::BUFR); impl::Messages msgs2 = read_msgs("bufr/obs0-3.504.bufr", Encoding::BUFR); auto msg1 = impl::Message::downcast(msgs1[0]); auto msg2 = impl::Message::downcast(msgs2[0]); f.tr->remove_all(); f.tr->import_message(*msg1, default_opts); f.tr->import_message(*msg2, default_opts); // Explicitly set the rep_memo variable that is added during export msg1->set_rep_memo(impl::Message::repmemo_from_type(msg1->type)); msg2->set_rep_memo(impl::Message::repmemo_from_type(msg2->type)); query.clear(); query.report = impl::Message::repmemo_from_type(msg1->type); // Warning: this test used to fail with older versions of MySQL. // See http://bugs.debian.org/cgi-bin/bugreport.cgi?bug=397597 impl::Messages msgs = dballe::tests::messages_from_db(f.tr, query); wassert(actual(msgs.size()) == 2u); // Compare the two dba_msg wassert(actual(diff_msg(msg1, msgs[0], "synop1")) == 0); wassert(actual(diff_msg(msg2, msgs[1], "synop2")) == 0); }); this->add_method("double", [](Fixture& f) { // Check that importing the same message twice works // msg1 has latitude 33.88 // msg2 has latitude 46.22 impl::Messages msgs1 = read_msgs("bufr/obs0-1.22.bufr", Encoding::BUFR); auto msg1 = impl::Message::downcast(msgs1[0]); f.tr->remove_all(); //auto t = db->transaction(); f.tr->import_message(*msg1, default_opts); f.tr->import_message(*msg1, default_opts); //t->commit(); // Explicitly set the rep_memo variable that is added during export msg1->set_rep_memo(impl::Message::repmemo_from_type(msg1->type)); core::Query query; query.report = impl::Message::repmemo_from_type(msg1->type); impl::Messages msgs = dballe::tests::messages_from_db(f.tr, query); wassert(actual(msgs.size()) == 1u); // Compare the two dba_msg wassert(actual(diff_msg(msg1, msgs[0], "synop1")) == 0); }); this->add_method("auto_repinfo", [](Fixture& f) { // Check automatic repinfo allocation core::Query query; impl::Messages msgs = read_msgs("bufr/generic-new-repmemo.bufr", Encoding::BUFR); auto msg = impl::Message::downcast(msgs[0]); f.tr->remove_all(); f.tr->import_message(*msg, default_opts); query.clear(); query.report = "enrico"; impl::Messages outmsgs = dballe::tests::messages_from_db(f.tr, query); wassert(actual(outmsgs.size()) == 1u); // Compare the two dba_msg wassert(actual(diff_msg(msg, outmsgs[0], "enrico")) == 0); }); this->add_method("station_only", [](Fixture& f) { // Check that a message that only contains station variables does get imported impl::Messages msgs = read_msgs("bufr/generic-onlystation.bufr", Encoding::BUFR); f.tr->remove_all(); f.tr->import_message(*msgs[0], default_opts); auto cur = f.tr->query_stations(core::Query()); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); auto vars = cur->get_values(); wassert(actual(vars.size()) == 5); wassert(actual(vars.var(WR_VAR(0, 1, 19)).format()) == "My beautifull station"); wassert(actual(vars.var(WR_VAR(0, 1, 194)).format()) == "generic"); wassert(actual(vars.var(WR_VAR(0, 5, 1)).format()) == "45.00000"); wassert(actual(vars.var(WR_VAR(0, 6, 1)).format()) == "10.00000"); wassert(actual(vars.var(WR_VAR(0, 7, 30)).format()) == "22.3"); }); this->add_method("station_only_no_vars", [](Fixture& f) { // Check that a message that only contains station variables does get imported impl::Messages msgs = read_msgs("bufr/arpa-station.bufr", Encoding::BUFR); f.tr->remove_all(); wassert(f.tr->import_message(*msgs[0], default_opts)); // Redo it with manually generated messages, this should not get imported { f.tr->remove_all(); impl::Message msg; msg.type = MessageType::GENERIC; msg.set_rep_memo("synop"); msg.set_latitude(44.53000); msg.set_longitude(11.30000); wassert(f.tr->import_message(msg, default_opts)); } // Same but with a datetime set. This should not get imported, but it // currently does because of a bug. I need to preserve the bug until // the software that relies on it has been migrated to use standard // DB-All.e features. { f.tr->remove_all(); impl::Message msg; msg.type = MessageType::GENERIC; msg.set_rep_memo("synop"); msg.set_latitude(44.53000); msg.set_longitude(11.30000); msg.set_datetime(Datetime(1000, 1, 1, 0, 0, 0)); #warning TODO: fix this test to give an error once we do not need to support this bug anymore //try { f.tr->import_message(msg, default_opts); //wassert(actual(false).istrue()); //} catch (error_notfound& e) { // ok. //} } }); this->add_method("import_dirty", [](Fixture& f) { auto opts = DBImportOptions::create(); opts->update_station = true; opts->overwrite = true; // Try importing into a dirty database, no attributes involved auto add_common = [](impl::Message& msg) { msg.type = MessageType::SYNOP; msg.set_rep_memo("synop"); msg.set_latitude(45.4); msg.set_longitude(11.2); msg.set_datetime(Datetime(2015, 4, 25, 12, 30, 45)); }; // Build test messages auto first = make_shared(); add_common(*first); first->set_block(1); // Station variable first->set_station(2); // Station variable first->set_temp_2m(280.1); // Data variable first->set_wet_temp_2m(275.8); // Data variable auto second = make_shared(); add_common(*second); second->set_block(5); // Station variable, different value second->set_station(2); // Station variable, same value second->set_height_station(101.0); // Station variable, new value second->set_temp_2m(281.1); // Data variable, different value second->set_wet_temp_2m(275.8); // Data variable, same value second->set_humidity(55.6); // Data variable, new value auto third = make_shared(); add_common(*third); third->set_block(6); // Station variable, different value third->set_station(2); // Station variable, same value third->set_height_station(101.0); // Station variable, same value as the second third->set_height_baro(102.0); // Station variable, new value third->set_temp_2m(282.1); // Data variable, different value third->set_wet_temp_2m(275.8); // Data variable, same value third->set_humidity(55.6); // Data variable, same value as the second third->set_dewpoint_2m(55.6); // Data variable, new value // Import the first message f.tr->remove_all(); f.tr->import_message(*first, *opts); // Export and check impl::Messages export_first = dballe::tests::messages_from_db(f.tr, "rep_memo=synop"); wassert(actual(export_first.size()) == 1); wassert(actual(diff_msg(first, export_first[0], "first")) == 0); // Import the second message f.tr->import_message(*second, *opts); // Export and check impl::Messages export_second = dballe::tests::messages_from_db(f.tr, "rep_memo=synop"); wassert(actual(export_second.size()) == 1); wassert(actual(diff_msg(second, export_second[0], "second")) == 0); // Try again with empty caches f.tr->clear_cached_state(); // Import the third message f.tr->import_message(*third, *opts); // Export and check impl::Messages export_third = dballe::tests::messages_from_db(f.tr, "rep_memo=synop"); wassert(actual(export_third.size()) == 1); wassert(actual(diff_msg(third, export_third[0], "third")) == 0); }); this->add_method("varlist", [](Fixture& f) { // Import filtering by varlist. See: #149 auto opts = DBImportOptions::create(); opts->varlist.push_back(WR_VAR(0, 12, 101)); opts->varlist.push_back(WR_VAR(0, 12, 103)); opts->varlist.push_back(WR_VAR(0, 13, 23)); impl::Messages inmsgs = read_msgs("bufr/gts-synop-linate.bufr", Encoding::BUFR); wassert(f.tr->import_messages(inmsgs, *opts)); core::Query query; auto cur = f.tr->query_data(query); unsigned count = 0; while (cur->next()) { wassert_true(std::find(opts->varlist.begin(), opts->varlist.end(), cur->get_varcode()) != opts->varlist.end()); ++count; } wassert(actual(count) == 3); }); } }; Tests tg2("db_import_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_import_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_import_v7_mysql", "MYSQL"); #endif } dballe-8.6/dballe/db/db-query-data-test.cc0000644000175000017500000003533713554564112015307 00000000000000#include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { static inline core::Query query_exact(const Datetime& dt) { core::Query query; query.dtrange = DatetimeRange(dt, dt); return query; } static inline core::Query query_min(const Datetime& dt) { core::Query query; query.dtrange = DatetimeRange(dt, Datetime()); return query; } static inline core::Query query_max(const Datetime& dt) { core::Query query; query.dtrange = DatetimeRange(Datetime(), dt); return query; } static inline core::Query query_minmax(const Datetime& min, const Datetime& max) { core::Query query; query.dtrange = DatetimeRange(min, max); return query; } struct DateHourDataSet : public TestDataSet { DateHourDataSet() { core::Data d; d.station.coords = Coords(12.34560, 76.54320); d.station.report = "synop"; d.level = Level(10, 11, 15, 22); d.trange = Trange(20, 111, 122); data["1"] = d; data["1"].datetime = Datetime(2013, 10, 30, 11); data["1"].values.set("B12101", 11.5); data["2"] = d; data["2"].datetime = Datetime(2013, 10, 30, 12); data["2"].values.set("B12101", 12.5); } }; struct DateDayDataSet : public TestDataSet { DateDayDataSet() { core::Data d; d.station.coords = Coords(12.34560, 76.54320); d.station.report = "synop"; d.level = Level(10, 11, 15, 22); d.trange = Trange(20, 111, 122); data["1"] = d; data["1"].datetime = Datetime(2013, 10, 23); data["1"].values.set("B12101", 23.5); data["2"] = d; data["2"].datetime = Datetime(2013, 10, 24); data["2"].values.set("B12101", 24.5); } }; #define TRY_QUERY(qstring, expected_count) wassert(actual(f.tr).try_data_query(qstring, expected_count)) template class OldFixtureTests : public FixtureTestCase> { typedef TransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; template class EmptyFixtureTests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; OldFixtureTests tg2("db_query_data1_v7_sqlite", "SQLITE"); EmptyFixtureTests tg4("db_query_data2_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ OldFixtureTests tg6("db_query_data1_v7_postgresql", "POSTGRESQL"); EmptyFixtureTests tg8("db_query_data2_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL OldFixtureTests tga("db_query_data1_v7_mysql", "MYSQL"); EmptyFixtureTests tgc("db_query_data2_v7_mysql", "MYSQL"); #endif template void OldFixtureTests::register_tests() { this->add_method("ana_id", [](Fixture& f) { char query[20]; snprintf(query, 20, "ana_id=%d", f.test_data.data["synop"].station.id); TRY_QUERY(query, 2); TRY_QUERY("ana_id=4242", 0); }); this->add_method("ana_context", [](Fixture& f) { // Query data in station context core::Query query; auto cur = f.tr->query_station_data(query); wassert(actual(cur->remaining()) == 10); }); this->add_method("year", [](Fixture& f) { // Datetime queries TRY_QUERY("year=1001", 0); TRY_QUERY("yearmin=1999", 0); TRY_QUERY("yearmin=1945", 4); TRY_QUERY("yearmax=1944", 0); TRY_QUERY("yearmax=1945", 4); TRY_QUERY("yearmax=2030", 4); TRY_QUERY("year=1944", 0); TRY_QUERY("year=1945", 4); TRY_QUERY("year=1946", 0); /* TRY_QUERY(i, DBA_KEY_MONTHMIN, 1); TRY_QUERY(i, DBA_KEY_MONTHMAX, 12); TRY_QUERY(i, DBA_KEY_MONTH, 5); */ /* TRY_QUERY(i, DBA_KEY_DAYMIN, 1); TRY_QUERY(i, DBA_KEY_DAYMAX, 12); TRY_QUERY(i, DBA_KEY_DAY, 5); */ /* TRY_QUERY(i, DBA_KEY_HOURMIN, 1); TRY_QUERY(i, DBA_KEY_HOURMAX, 12); TRY_QUERY(i, DBA_KEY_HOUR, 5); */ /* TRY_QUERY(i, DBA_KEY_MINUMIN, 1); TRY_QUERY(i, DBA_KEY_MINUMAX, 12); TRY_QUERY(i, DBA_KEY_MIN, 5); */ /* TRY_QUERY(i, DBA_KEY_SECMIN, 1); TRY_QUERY(i, DBA_KEY_SECMAX, 12); TRY_QUERY(i, DBA_KEY_SEC, 5); */ }); this->add_method("block_station", [](Fixture& f) { // Block and station queries TRY_QUERY("block=1", 4); TRY_QUERY("block=2", 0); TRY_QUERY("station=52", 4); TRY_QUERY("station=53", 0); }); this->add_method("ana_filter", [](Fixture& f) { // ana_filter queries TRY_QUERY("ana_filter=block=1", 4); TRY_QUERY("ana_filter=B01001=1", 4); TRY_QUERY("ana_filter=block>1", 0); TRY_QUERY("ana_filter=B01001>1", 0); TRY_QUERY("ana_filter=block<=1", 4); TRY_QUERY("ana_filter=B01001<=1", 4); TRY_QUERY("ana_filter=0<=B01001<=2", 4); TRY_QUERY("ana_filter=1<=B01001<=1", 4); TRY_QUERY("ana_filter=2<=B01001<=4", 0); }); this->add_method("data_filter", [](Fixture& f) { // data_filter queries TRY_QUERY("data_filter=B01011=DB-All.e!", 1); TRY_QUERY("data_filter=B01012<300", 0); TRY_QUERY("data_filter=B01012<=300", 1); TRY_QUERY("data_filter=B01012=300", 1); TRY_QUERY("data_filter=B01012>=300", 2); TRY_QUERY("data_filter=B01012>300", 1); TRY_QUERY("data_filter=B01012<400", 1); TRY_QUERY("data_filter=B01012<=400", 2); }); this->add_method("latlon", [](Fixture& f) { // latitude/longitude queries TRY_QUERY("latmin=11.0", 4); TRY_QUERY("latmin=12.34560", 4); TRY_QUERY("latmin=13.0", 0); TRY_QUERY("latmax=11.0", 0); TRY_QUERY("latmax=12.34560", 4); TRY_QUERY("latmax=13.0", 4); TRY_QUERY("latmin=0, latmax=20", 4); TRY_QUERY("latmin=-90, latmax=20", 4); TRY_QUERY("latmin=-90, latmax=0", 0); TRY_QUERY("latmin=10, latmax=90", 4); TRY_QUERY("latmin=45, latmax=90", 0); TRY_QUERY("latmin=-90, latmax=90", 4); TRY_QUERY("lonmin=75, lonmax=77", 4); TRY_QUERY("lonmin=76.54320, lonmax=76.54320", 4); TRY_QUERY("lonmin=76.54330, lonmax=77.", 0); TRY_QUERY("lonmin=77., lonmax=76.54330", 4); TRY_QUERY("lonmin=77., lonmax=76.54320", 4); TRY_QUERY("lonmin=77., lonmax=-10", 0); TRY_QUERY("lonmin=0., lonmax=360.", 0); TRY_QUERY("lonmin=76.54320, lonmax=436.54320", 4); TRY_QUERY("lonmin=-180., lonmax=180.", 0); }); this->add_method("mobile", [](Fixture& f) { // fixed/mobile queries TRY_QUERY("mobile=0", 4); TRY_QUERY("mobile=1", 0); }); // ident queries // FIXME: we currently have no mobile station data in the samples //TRY_QUERY(c, DBA_KEY_IDENT_SELECT, "pippo"); this->add_method("timerange", [](Fixture& f) { // timerange queries TRY_QUERY("pindicator=20", 4); TRY_QUERY("pindicator=21", 0); TRY_QUERY("p1=111", 4); TRY_QUERY("p1=112", 0); TRY_QUERY("p2=121", 0); TRY_QUERY("p2=122", 2); TRY_QUERY("p2=123", 2); }); this->add_method("level", [](Fixture& f) { // level queries TRY_QUERY("leveltype1=10", 4); TRY_QUERY("leveltype1=11", 0); TRY_QUERY("leveltype2=15", 4); TRY_QUERY("leveltype2=16", 0); TRY_QUERY("l1=11", 4); TRY_QUERY("l1=12", 0); TRY_QUERY("l2=22", 4); TRY_QUERY("l2=23", 0); }); this->add_method("varcode", [](Fixture& f) { // varcode queries TRY_QUERY("var=B01011", 2); TRY_QUERY("var=B01012", 2); TRY_QUERY("var=B01013", 0); }); this->add_method("report", [](Fixture& f) { // report queries TRY_QUERY("rep_memo=synop", 2); TRY_QUERY("rep_memo=metar", 2); TRY_QUERY("rep_memo=temp", 0); }); this->add_method("priority", [](Fixture& f) { // report priority queries TRY_QUERY("priority=101", 2); TRY_QUERY("priority=81", 2); TRY_QUERY("priority=102", 0); TRY_QUERY("priomin=70", 4); TRY_QUERY("priomin=80", 4); TRY_QUERY("priomin=90", 2); TRY_QUERY("priomin=100", 2); TRY_QUERY("priomin=110", 0); TRY_QUERY("priomax=70", 0); TRY_QUERY("priomax=81", 2); TRY_QUERY("priomax=100", 2); TRY_QUERY("priomax=101", 4); TRY_QUERY("priomax=110", 4); }); } template void EmptyFixtureTests::register_tests() { this->add_method("datetime1", [](Fixture& f) { // Check datetime queries, with data that only differs by its hour DateHourDataSet test_data; wassert(f.populate(test_data)); // Valid hours: 11 and 12 // Exact match wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 30, 10)), 0)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 30, 11)), 1)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 30, 12)), 1)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 30, 13)), 0)); // Datemin match wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 30, 10)), 2)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 30, 11)), 2)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 30, 12)), 1)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 30, 13)), 0)); // Datemax match wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 30, 13)), 2)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 30, 12)), 2)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 30, 11)), 1)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 30, 10)), 0)); // Date min-max match wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 10), Datetime(2013, 10, 30, 13)), 2)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 11), Datetime(2013, 10, 30, 12)), 2)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 10), Datetime(2013, 10, 30, 11)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 12), Datetime(2013, 10, 30, 13)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 9), Datetime(2013, 10, 30, 10)), 0)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 30, 13), Datetime(2013, 10, 30, 14)), 0)); }); this->add_method("datetime2", [](Fixture& f) { // Check datetime queries, with data that only differs by its day DateDayDataSet test_data; wassert(f.populate(test_data)); // Valid days: 23 and 24 // Exact match wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 22)), 0)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 23)), 1)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 24)), 1)); wassert(actual(f.tr).try_data_query(query_exact(Datetime(2013, 10, 25)), 0)); // Datemin match wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 22)), 2)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 23)), 2)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 24)), 1)); wassert(actual(f.tr).try_data_query(query_min(Datetime(2013, 10, 25)), 0)); // Datemax match wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 25)), 2)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 24)), 2)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 23)), 1)); wassert(actual(f.tr).try_data_query(query_max(Datetime(2013, 10, 22)), 0)); // Date min-max match wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 22), Datetime(2013, 10, 25)), 2)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 23), Datetime(2013, 10, 24)), 2)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 23), Datetime(2013, 10, 23)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 24), Datetime(2013, 10, 24)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 22), Datetime(2013, 10, 23)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 24), Datetime(2013, 10, 25)), 1)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 21), Datetime(2013, 10, 22)), 0)); wassert(actual(f.tr).try_data_query(query_minmax(Datetime(2013, 10, 25), Datetime(2013, 10, 26)), 0)); }); this->add_method("query_ordering", [](Fixture& f) { auto insert = [&](const char* str) { core::Data data; data.set_from_test_string(str); wassert(f.tr->insert_data(data)); return data; }; auto vals01 = insert("lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=synop, B12101=280.15"); auto vals02 = insert("lat=2, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=synop, B12101=280.15"); auto vals03 = insert("lat=1, lon=1, year=2001, leveltype1=1, pindicator=1, rep_memo=synop, B12101=280.15"); auto vals04 = insert("lat=1, lon=1, year=2000, leveltype1=2, pindicator=1, rep_memo=synop, B12101=280.15"); auto vals05 = insert("lat=1, lon=1, year=2000, leveltype1=1, pindicator=2, rep_memo=synop, B12101=280.15"); auto vals06 = insert("lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=metar, B12101=280.15"); auto vals07 = insert("lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=synop, B12103=280.15"); auto cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 7); switch (DB::format) { case Format::V7: // v7: ana_id(coords, ident, report), datetime, level, trange, code wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals01)); // lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=a, B12101=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals07)); // lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=a, B12103=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals05)); // lat=1, lon=1, year=2000, leveltype1=1, pindicator=2, rep_memo=a, B12101=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals04)); // lat=1, lon=1, year=2000, leveltype1=2, pindicator=1, rep_memo=a, B12101=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals03)); // lat=1, lon=1, year=2001, leveltype1=1, pindicator=1, rep_memo=a, B12101=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals02)); // lat=2, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=a, B12101=280.15 wassert(actual(cur->next())); wassert(actual(cur).data_matches(vals06)); // lat=1, lon=1, year=2000, leveltype1=1, pindicator=1, rep_memo=b, B12101=280.15 break; default: error_unimplemented::throwf("cannot run this test on a database of format %d", (int)DB::format); } }); } } dballe-8.6/dballe/db/summary-test.cc0000644000175000017500000002007513554564112014336 00000000000000#define _DBALLE_TEST_CODE #include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "summary.h" #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg1("db_summary_v7_sqlite_summary", "SQLITE"); Tests tg2("db_summary_v7_sqlite_dbsummary", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg3("db_summary_v7_postgresql_summary", "POSTGRESQL"); Tests tg4("db_summary_v7_postgresql_dbsummary", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg5("db_summary_v7_mysql_summary", "MYSQL"); Tests tg6("db_summary_v7_mysql_dbsummary", "MYSQL"); #endif void station_id_isset(const Station& station) {} void station_id_isset(const DBStation& station) { wassert(actual(station.id) != MISSING_INT); } std::unique_ptr other_summary(const BaseSummary&) { return std::unique_ptr(new Summary); } std::unique_ptr other_summary(const BaseSummary&) { return std::unique_ptr(new DBSummary); } Station other_station(const DBStation& station) { Station res(station); return res; } DBStation other_station(const Station& station) { DBStation res; res.report = station.report; res.coords = station.coords; res.ident = station.ident; return res; } void set_query_station(core::Query& query, const Station& station) { query.report = station.report; query.set_latrange(LatRange(station.coords.lat, station.coords.lat)); query.set_lonrange(LonRange(station.coords.lon, station.coords.lon)); query.ident = station.ident; } void set_query_station(core::Query& query, const DBStation& station) { query.ana_id = station.id; } template void Tests::register_tests() { this->add_method("summary", [](Fixture& f) { // Test building a summary and checking if it supports queries OldDballeTestDataSet test_data; wassert(f.populate(test_data)); BaseSummary s; wassert_true(s.datetime_min().is_missing()); wassert_true(s.datetime_max().is_missing()); wassert(actual(s.data_count()) == 0u); // Build the whole db summary core::Query query; query.query = "details"; auto cur = f.tr->query_summary(query); while (cur->next()) s.add_cursor(*cur); // Check its contents wassert(actual(s.stations().size()) == 2); wassert(station_id_isset(s.stations().begin()->station)); wassert(actual(s.levels().size()) == 1); wassert(actual(s.tranges().size()) == 2); wassert(actual(s.varcodes().size()) == 2); wassert(actual(s.datetime_min()) == Datetime(1945, 4, 25, 8)); wassert(actual(s.datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(s.data_count()) == 4); set_query_station(query, s.stations().begin()->station); BaseSummary s1; s1.add_filtered(s, query); wassert(actual(s1.stations().size()) == 1); // wassert(actual(s1.stations().begin()->station.id) == query.ana_id); BaseSummary s2; cur = s.query_summary(query); while (cur->next()) s2.add_cursor(*cur); wassert(actual(s2.stations().size()) == 1); }); this->add_method("summary_msg", [](Fixture& f) { BaseSummary s; // Summarise a message impl::Messages msgs = dballe::tests::read_msgs("bufr/synop-rad1.bufr", Encoding::BUFR, "accurate"); s.add_messages(msgs); // Check its contents wassert(actual(s.stations().size()) == 25); wassert(actual(s.levels().size()) == 37); wassert(actual(s.tranges().size()) == 9); wassert(actual(s.varcodes().size()) == 39); wassert(actual(s.datetime_min()) == Datetime(2015, 3, 5, 3)); wassert(actual(s.datetime_max()) == Datetime(2015, 3, 5, 3)); wassert(actual(s.data_count()) == 1095); }); this->add_method("merge_entries", [](Fixture& f) { STATION station; station.report = "test"; station.coords = Coords(44.5, 11.5); summary::VarDesc vd(Level(1), Trange::instant(), WR_VAR(0, 1, 112)); DatetimeRange dtrange(Datetime(2018, 1, 1), Datetime(2018, 7, 1)); BaseSummary summary; wassert(actual(summary.data_count()) == 0u); summary.add(station, vd, dtrange, 12u); wassert(actual(summary.data_count()) == 12u); summary.add(station, vd, dtrange, 12u); wassert(actual(summary.data_count()) == 24u); wassert(actual(summary.stations().size()) == 1); wassert(actual(summary.stations().begin()->size()) == 1); summary.add(station, vd, dtrange, 12u); station.report = "test1"; summary.add(station, vd, dtrange, 12u); summary.add(station, vd, dtrange, 12u); wassert(actual(summary.stations().size()) == 2); wassert(actual(summary.stations().begin()->size()) == 1); wassert(actual(summary.stations().rbegin()->size()) == 1); wassert(actual(summary.data_count()) == 36u + 24u); }); this->add_method("merge_summaries", [](Fixture& f) { BaseSummary summary; STATION station; station.report = "test"; station.coords = Coords(44.5, 11.5); summary::VarDesc vd(Level(1), Trange::instant(), WR_VAR(0, 1, 112)); DatetimeRange dtrange(Datetime(2018, 1, 1), Datetime(2018, 7, 1)); summary.add(station, vd, dtrange, 12u); BaseSummary summary1; summary1.add(station, vd, dtrange, 3u); auto summary2 = other_summary(summary); summary2->add(other_station(station), vd, dtrange, 2u); summary.add_summary(summary1); summary.add_summary(*summary2); wassert(actual(summary.data_count()) == 17u); }); this->add_method("json_summary", [](Fixture& f) { STATION station; station.report = "test"; station.coords = Coords(44.5, 11.5); summary::VarDesc vd(Level(1), Trange::instant(), WR_VAR(0, 1, 112)); DatetimeRange dtrange(Datetime(2018, 1, 1), Datetime(2018, 7, 1)); core::Query query; query.report = "synop"; BaseSummary summary; summary.add(station, vd, dtrange, 12); vd.varcode = WR_VAR(0, 1, 113); summary.add(station, vd, dtrange, 12); std::stringstream json; core::JSONWriter writer(json); summary.to_json(writer); wassert(actual(json.str()) == R"({"e":[{"s":{"r":"test","c":[4450000,1150000]},"v":[{"l":[1,null,null,null],"t":[254,0,0],"v":368,"d":[[2018,1,1,0,0,0],[2018,7,1,0,0,0]],"c":12},{"l":[1,null,null,null],"t":[254,0,0],"v":369,"d":[[2018,1,1,0,0,0],[2018,7,1,0,0,0]],"c":12}]}]})"); json.seekg(0); core::json::Stream in(json); BaseSummary summary1; wassert(summary1.load_json(in)); wassert_true(summary == summary1); wassert(actual(summary.stations().size()) == summary1.stations().size()); wassert_true(summary.stations() == summary1.stations()); wassert_true(summary.reports() == summary1.reports()); wassert_true(summary.levels() == summary1.levels()); wassert_true(summary.tranges() == summary1.tranges()); wassert_true(summary.varcodes() == summary1.varcodes()); wassert_true(summary.datetime_min() == summary1.datetime_min()); wassert_true(summary.datetime_max() == summary1.datetime_max()); wassert_true(summary.data_count() == summary1.data_count()); // Check that load does merge json.seekg(0); core::json::Stream in1(json); wassert(summary1.load_json(in1)); wassert(actual(summary.stations().size()) == summary1.stations().size()); wassert_true(summary.reports() == summary1.reports()); wassert_true(summary.levels() == summary1.levels()); wassert_true(summary.tranges() == summary1.tranges()); wassert_true(summary.varcodes() == summary1.varcodes()); wassert_true(summary.datetime_min() == summary1.datetime_min()); wassert_true(summary.datetime_max() == summary1.datetime_max()); wassert_true(summary.data_count() * 2 == summary1.data_count()); }); } } dballe-8.6/dballe/db/db.cc0000644000175000017500000001323013554564112012244 00000000000000#include "config.h" #include "db.h" #include "v7/db.h" #include "dballe/sql/sql.h" #include "dballe/sql/sqlite.h" #include "dballe/message.h" #include "dballe/file.h" #include #include #include using namespace dballe::db; using namespace std; using namespace wreport; namespace dballe { namespace db { void CursorStationData::insert_attrs(const Values& attrs) { get_transaction()->attr_insert_station(attr_reference_id(), attrs); } void CursorStationData::remove_attrs(const db::AttrList& attrs) { get_transaction()->attr_remove_station(attr_reference_id(), attrs); } void CursorData::insert_attrs(const Values& attrs) { get_transaction()->attr_insert_data(attr_reference_id(), attrs); } void CursorData::remove_attrs(const db::AttrList& attrs) { get_transaction()->attr_remove_data(attr_reference_id(), attrs); } static Format default_format = Format::V7; std::string format_format(Format format) { switch (format) { case Format::V5: return "V5"; case Format::V6: return "V6"; case Format::MEM: return "MEM"; case Format::MESSAGES: return "MESSAGES"; case Format::V7: return "V7"; default: return "unknown format " + std::to_string((int)format); } } Format format_parse(const std::string& str) { if (str == "V7") return Format::V7; if (str == "V6") return Format::V6; if (str == "V5") return Format::V5; if (str == "MEM") return Format::MEM; if (str == "MESSAGES") return Format::MESSAGES; error_consistency::throwf("unsupported database format: '%s'", str.c_str()); } Format DB::get_default_format() { return default_format; } void DB::set_default_format(Format format) { default_format = format; } bool DB::is_url(const char* str) { if (strncmp(str, "mem:", 4) == 0) return true; if (strncmp(str, "sqlite:", 7) == 0) return true; if (strncmp(str, "postgresql:", 11) == 0) return true; if (strncmp(str, "mysql:", 6) == 0) return true; if (strncmp(str, "test:", 5) == 0) return true; return false; } std::shared_ptr DB::create(unique_ptr conn) { // Autodetect format Format format = default_format; const char* format_override = getenv("DBA_DB_FORMAT"); if (format_override) format = format_parse(format_override); bool found = true; // Try with reading it from the settings table string version = conn->get_setting("version"); if (version == "V5") format = Format::V5; else if (version == "V6") format = Format::V6; else if (version == "V7") format = Format::V7; else if (version == "") found = false;// Some other key exists, but the version has not been set else error_consistency::throwf("unsupported database version: '%s'", version.c_str()); // If it failed, try looking at the existing table structure if (!found) { if (conn->has_table("lev_tr")) format = Format::V6; else if (conn->has_table("context")) format = Format::V5; } switch (format) { case Format::V5: throw error_unimplemented("V5 format is not supported anymore by this version of DB-All.e"); case Format::V6: throw error_unimplemented("V6 format is not supported anymore by this version of DB-All.e"); case Format::V7: return static_pointer_cast(make_shared(move(conn))); default: error_consistency::throwf("requested unknown format %d", (int)format); } } shared_ptr DB::connect_from_file(const char* pathname) { unique_ptr conn(new sql::SQLiteConnection); conn->open_file(pathname); return create(unique_ptr(conn.release())); } shared_ptr DB::connect_memory() { sql::SQLiteConnection* sqlite_conn; unique_ptr conn(sqlite_conn = new sql::SQLiteConnection); sqlite_conn->open_memory(); auto res = static_pointer_cast(make_shared(move(conn))); res->reset(); return res; } const char* DB::default_repinfo_file() { const char* repinfo_file = getenv("DBA_REPINFO"); if (repinfo_file == 0 || repinfo_file[0] == 0) repinfo_file = TABLE_DIR "/repinfo.csv"; return repinfo_file; } void DB::attr_query_station(int data_id, std::function)>&& dest) { auto t = dynamic_pointer_cast(transaction()); t->attr_query_station(data_id, move(dest)); t->commit(); } void DB::attr_query_data(int data_id, std::function)>&& dest) { auto t = dynamic_pointer_cast(transaction()); t->attr_query_data(data_id, move(dest)); t->commit(); } void DB::attr_insert_station(int data_id, const Values& attrs) { auto t = dynamic_pointer_cast(transaction()); t->attr_insert_station(data_id, attrs); t->commit(); } void DB::attr_insert_data(int data_id, const Values& attrs) { auto t = dynamic_pointer_cast(transaction()); t->attr_insert_data(data_id, attrs); t->commit(); } void DB::attr_remove_station(int data_id, const db::AttrList& attrs) { auto t = dynamic_pointer_cast(transaction()); t->attr_remove_station(data_id, attrs); t->commit(); } void DB::attr_remove_data(int data_id, const db::AttrList& attrs) { auto t = dynamic_pointer_cast(transaction()); t->attr_remove_data(data_id, attrs); t->commit(); } void DB::dump(FILE* out) { auto t = dynamic_pointer_cast(transaction()); t->dump(out); t->rollback(); } void DB::print_info(FILE* out) { fprintf(out, "Format: %s\n", format_format(format()).c_str()); } } } dballe-8.6/dballe/db/tests.h0000644000175000017500000001554513554564112012676 00000000000000#include #include #include #include #include #include #include #include #include #include namespace dballe { namespace tests { impl::Messages messages_from_db(std::shared_ptr tr, const dballe::Query& query); impl::Messages messages_from_db(std::shared_ptr tr, const char* query); /// Base for datasets used to populate test databases struct TestDataSet { /// Arbitrarily named station values std::map stations; /// Arbitrarily named data values std::map data; TestDataSet() {} virtual ~TestDataSet() {} void populate_db(DB& db); virtual void populate_transaction(Transaction& tr); }; struct EmptyTestDataset { void populate_db(); }; /// Test fixture used by old DB-All.e db tests struct OldDballeTestDataSet : public TestDataSet { OldDballeTestDataSet(); }; bool has_driver(const std::string& backend); struct V7DB { typedef db::v7::DB DB; typedef db::v7::Transaction TR; static const auto format = db::Format::V7; static std::shared_ptr create_db(const std::string& backend, bool wipe); }; template struct BaseDBFixture : public Fixture { std::string backend; std::shared_ptr db; BaseDBFixture(const char* backend); ~BaseDBFixture(); void test_setup(); virtual void create_db(); bool has_driver(); }; template struct EmptyTransactionFixture : public BaseDBFixture { using BaseDBFixture::BaseDBFixture; std::shared_ptr tr; void test_setup(); void test_teardown(); void populate(TestDataSet& data_set); }; template struct TransactionFixture : public EmptyTransactionFixture { using EmptyTransactionFixture::EmptyTransactionFixture; TestData test_data; void create_db() override { EmptyTransactionFixture::create_db(); wassert(test_data.populate_db(*this->db)); } }; template struct DBFixture : public BaseDBFixture { using BaseDBFixture::BaseDBFixture; void test_setup(); void populate_database(TestDataSet& data_set); }; struct ActualCursor : public Actual { using Actual::Actual; /// Check cursor context after a query_stations void station_keys_match(const DBStation& expected); /// Check cursor data context after a query_data void data_context_matches(const Data& expected); /// Check cursor data variable after a query_data void data_var_matches(const Data& expected, wreport::Varcode code) { data_var_matches(core::Data::downcast(expected).values.var(code)); } /// Check cursor data variable(s) after a query_data void data_var_matches(const DBValues& expected) { if (auto c = dynamic_cast(&_actual)) { DBValues actual_values = c->get_values(); if (!actual_values.vars_equal(expected)) // Quick hack to get proper formatting of mismatch wassert(actual(c->get_values()) == expected); } else if (auto c = dynamic_cast(&_actual)) data_var_matches(expected.var(c->get_varcode())); else if (auto c = dynamic_cast(&_actual)) data_var_matches(expected.var(c->get_varcode())); else throw wreport::error_consistency("cannot call data_var_matches on this kind of cursor"); } /// Check cursor data variable after a query_data void data_var_matches(const Values& expected, wreport::Varcode code) { data_var_matches(expected.var(code)); } /// Check cursor data variable after a query_data void data_var_matches(const wreport::Var& expected); /// Check cursor data context and variable after a query_data void data_matches(const Data& ds) { if (auto c = dynamic_cast(&_actual)) data_matches(ds, c->get_varcode()); else if (auto c = dynamic_cast(&_actual)) data_matches(ds, c->get_varcode()); else throw wreport::error_consistency("cannot call data_matches on this kind of cursor"); } /// Check cursor data context and variable after a query_data void data_matches(const Data& ds, wreport::Varcode code); }; typedef std::function result_checker; template struct ActualDB : public Actual> { using Actual>::Actual; /// Check cursor data context anda variable after a query_data void try_data_query(const std::string& query, unsigned expected); /// Check cursor data context anda variable after a query_data void try_data_query(const Query& query, unsigned expected); /// Check results of a station query void try_station_query(const std::string& query, unsigned expected); /// Check results of a summary query void try_summary_query(const std::string& query, unsigned expected, result_checker checker=nullptr); }; inline ActualCursor actual(dballe::Cursor& actual) { return ActualCursor(actual); } inline ActualCursor actual(dballe::CursorStation& actual) { return ActualCursor(actual); } inline ActualCursor actual(dballe::CursorStationData& actual) { return ActualCursor(actual); } inline ActualCursor actual(dballe::CursorData& actual) { return ActualCursor(actual); } inline ActualCursor actual(dballe::CursorSummary& actual) { return ActualCursor(actual); } inline ActualCursor actual(std::unique_ptr& actual) { return ActualCursor(*actual); } inline ActualCursor actual(std::unique_ptr& actual) { return ActualCursor(*actual); } inline ActualCursor actual(std::unique_ptr& actual) { return ActualCursor(*actual); } inline ActualCursor actual(std::unique_ptr& actual) { return ActualCursor(*actual); } inline ActualCursor actual(std::unique_ptr& actual) { return ActualCursor(*actual); } inline ActualDB actual(std::shared_ptr actual) { return ActualDB(actual); } ActualDB actual(std::shared_ptr actual); inline ActualDB actual(std::shared_ptr actual) { return ActualDB(actual); } ActualDB actual(std::shared_ptr actual); extern template class BaseDBFixture; extern template class DBFixture; extern template class EmptyTransactionFixture; extern template class ActualDB; extern template class ActualDB; } } dballe-8.6/dballe/db/explorer.cc0000644000175000017500000001270013554564112013520 00000000000000#define _DBALLE_LIBRARY_CODE #include "explorer.h" #include "dballe/core/query.h" #include "dballe/core/json.h" #include using namespace std; using namespace dballe; namespace dballe { namespace db { #if 0 namespace { template bool has_db() { return false; } template<> bool has_db() { return true; } } #endif template BaseExplorer::BaseExplorer() { } template BaseExplorer::~BaseExplorer() { delete _global_summary; delete _active_summary; } template const dballe::db::BaseSummary& BaseExplorer::global_summary() const { if (!_global_summary) throw std::runtime_error("global summary is not available, call rebuild or update first"); return *_global_summary; } template const dballe::db::BaseSummary& BaseExplorer::active_summary() const { if (!_active_summary) throw std::runtime_error("active summary is not available, call rebuild or update first"); return *_active_summary; } template const dballe::Query& BaseExplorer::get_filter() const { return filter; } template void BaseExplorer::set_filter(const dballe::Query& query) { filter = core::Query::downcast(query); if (_global_summary) update_active_summary(); } template typename BaseExplorer::Update BaseExplorer::rebuild() { delete _global_summary; _global_summary = new db::BaseSummary; delete _active_summary; _active_summary = nullptr; return Update(this); } template typename BaseExplorer::Update BaseExplorer::update() { if (!_global_summary) _global_summary = new db::BaseSummary; delete _active_summary; _active_summary = nullptr; return Update(this); } template void BaseExplorer::update_active_summary() { unique_ptr> new_active_summary(new db::BaseSummary); new_active_summary->add_filtered(*_global_summary, filter); _active_summary = new_active_summary.release(); } template void BaseExplorer::to_json(core::JSONWriter& writer) const { writer.start_mapping(); writer.add("summary"); _global_summary->to_json(writer); writer.end_mapping(); } template BaseExplorer::Update::Update(BaseExplorer* explorer) : explorer(explorer) {} template BaseExplorer::Update::Update() {} template BaseExplorer::Update::Update(Update&& o) : explorer(o.explorer) { o.explorer = nullptr; } template BaseExplorer::Update::~Update() { commit(); } template void BaseExplorer::Update::commit() { if (!explorer) return; explorer->update_active_summary(); explorer = nullptr; } template typename BaseExplorer::Update& BaseExplorer::Update::operator=(Update&& o) { if (&o == this) return *this; explorer = o.explorer; o.explorer = nullptr; return *this; } template void BaseExplorer::Update::add_db(dballe::db::Transaction& tr) { core::Query query; query.query = "details"; auto cur = tr.query_summary(query); add_cursor(*cur); } template void BaseExplorer::Update::add_cursor(dballe::CursorSummary& cur) { while (cur.next()) explorer->_global_summary->add_cursor(cur); } template void BaseExplorer::Update::add_json(core::json::Stream& in) { in.parse_object([&](const std::string& key) { if (key == "summary") explorer->_global_summary->load_json(in); else throw core::JSONParseException("unsupported key \"" + key + "\" for db::Explorer"); }); } template template void BaseExplorer::Update::add_explorer(const BaseExplorer& explorer) { this->explorer->_global_summary->add_summary(explorer.global_summary()); } template<> template<> void BaseExplorer::Update::add_explorer(const BaseExplorer& explorer) { if (this->explorer == &explorer) wreport::error_consistency::throwf("Adding an Explorer to itself is not supported"); this->explorer->_global_summary->add_summary(explorer.global_summary()); } template<> template<> void BaseExplorer::Update::add_explorer(const BaseExplorer& explorer) { if (this->explorer == &explorer) wreport::error_consistency::throwf("Adding an Explorer to itself is not supported"); this->explorer->_global_summary->add_summary(explorer.global_summary()); } template void BaseExplorer::Update::add_message(const dballe::Message& message) { this->explorer->_global_summary->add_message(message); } template void BaseExplorer::Update::add_messages(const std::vector>& messages) { this->explorer->_global_summary->add_messages(messages); } template class BaseExplorer; template void BaseExplorer::Update::add_explorer(const BaseExplorer&); template class BaseExplorer; template void BaseExplorer::Update::add_explorer(const BaseExplorer&); } } dballe-8.6/dballe/db/db-misc-test.cc0000644000175000017500000010030013554564112014145 00000000000000#include "dballe/db/tests.h" #include "v7/db.h" #include "v7/transaction.h" #include "config.h" #include #include using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct NavileDataSet : public TestDataSet { NavileDataSet() { stations["synop"].station.coords = Coords(44.5008, 11.3288); stations["synop"].station.report = "synop"; stations["synop"].values.set("B07030", 78); // Height } }; unsigned run_attr_query_data(std::shared_ptr tr, int data_id, Values& dest) { unsigned count = 0; tr->attr_query_data(data_id, [&](unique_ptr var) { dest.set(move(var)); ++count; }); return count; } template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; template class CommitTests : public FixtureTestCase> { typedef DBFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg2("db_misc_tr_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_misc_tr_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_misc_tr_v7_mysql", "MYSQL"); #endif CommitTests ct2("db_misc_db_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ CommitTests ct4("db_misc_db_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL CommitTests ct6("db_misc_db_v7_mysql", "MYSQL"); #endif template void Tests::register_tests() { this->add_method("insert", [](Fixture& f) { // Test a simple insert round trip // Insert some data NavileDataSet ds; ds.data["synop"].station = ds.stations["synop"].station; ds.data["synop"].datetime = Datetime(2013, 10, 16, 10); ds.data["synop"].level = Level(1, 0, 0); ds.data["synop"].trange = Trange::instant(); ds.data["synop"].values.set(WR_VAR(0, 12, 101), 16.5); wassert(f.populate(ds)); Values attrs; attrs.set("B33007", 50); wassert(f.tr->attr_insert_data(ds.data["synop"].values.value(WR_VAR(0, 12, 101)).data_id, attrs)); // Query and verify the station data { auto cur = f.tr->query_stations(core::Query()); wassert(actual(cur->remaining()) == 1); cur->next(); wassert(actual(cur).station_keys_match(ds.stations["synop"].station)); wassert(actual(cur).data_var_matches(ds.stations["synop"].values)); } // Query and verify the measured data { auto cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 1); cur->next(); wassert(actual(cur).data_context_matches(ds.data["synop"])); wassert(actual(cur).data_var_matches(ds.data["synop"], WR_VAR(0, 12, 101))); } // Query and verify attributes { int count = 0; unique_ptr attr; wassert(f.tr->attr_query_data(ds.data["synop"].values.value(WR_VAR(0, 12, 101)).data_id, [&](std::unique_ptr&& var) { ++count; attr = move(var); })); wassert(actual(count) == 1); wassert(actual(attr->code()) == WR_VAR(0, 33, 7)); wassert(actual(attr->enq(MISSING_INT)) == 50); } }); this->add_method("insert_perms", [](Fixture& f) { // Test insert OldDballeTestDataSet oldf; impl::DBInsertOptions opts; opts.can_replace = false; opts.can_add_stations = false; // Check if adding a nonexisting station when not allowed causes an error try { f.tr->insert_data(oldf.data["synop"], opts); throw TestFailed("error_consistency should have been thrown"); } catch (error_consistency& e) { wassert(actual(e.what()).contains("insert a station entry when it is forbidden")); } catch (error_notfound& e) { wassert(actual(e.what()).contains("station not found")); } wassert(actual(oldf.data["synop"].station.id) == MISSING_INT); wassert(actual(oldf.data["synop"].values.value("B01011").data_id) == MISSING_INT); wassert(actual(oldf.data["synop"].values.value("B01012").data_id) == MISSING_INT); oldf.data["synop"].clear_ids(); // Insert the record opts.can_add_stations = true; wassert(f.tr->insert_data(oldf.data["synop"], opts)); oldf.data["synop"].clear_ids(); // Check if duplicate updates are allowed by insert opts.can_replace = true; opts.can_add_stations = false; wassert(f.tr->insert_data(oldf.data["synop"], opts)); oldf.data["synop"].clear_ids(); // Check if overwrites are trapped by insert_new oldf.data["synop"].values.set("B01011", "DB-All.e?"); opts.can_replace = opts.can_add_stations = false; try { f.tr->insert_data(oldf.data["synop"], opts); throw TestFailed("wreport::error should have been thrown"); } catch (wreport::error& e) { wassert(actual(e.what()).matches("refusing to overwrite existing data|cannot replace an existing value|Duplicate entry")); } }); this->add_method("insert_twice", [](Fixture& f) { // Test double station insert OldDballeTestDataSet oldf; // Insert the record twice wassert(f.tr->insert_data(oldf.data["synop"])); // This should fail, refusing to replace station info oldf.data["synop"].values.set("B01011", "DB-All.e?"); try { f.tr->insert_data(oldf.data["synop"]); throw TestFailed("wreport::error should have been thrown"); } catch (wreport::error& e) { wassert(actual(e.what()).matches("refusing to overwrite existing data|cannot replace an existing value|Duplicate entry")); } }); this->add_method("query_station", [](Fixture& f) { // Test station query OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // Iterate the station database auto cur = f.tr->query_stations(core::Query()); switch (DB::format) { case Format::V7: { bool have_synop = false; bool have_metar = false; // Memdb and V7 have one station entry per (lat, lon, ident, network) wassert(actual(cur->remaining()) == 2); for (unsigned i = 0; i < 2; ++i) { wassert(actual(cur->next()).istrue()); DBStation station = cur->get_station(); if (station.report == "synop") { wassert(actual(station.coords) == Coords(12.34560, 76.54320)); wassert(actual(station.ident.is_missing()).istrue()); wassert(actual(cur).station_keys_match(oldf.stations["synop"].station)); have_synop = true; } if (station.report == "metar") { wassert(actual(station.coords) == Coords(12.34560, 76.54320)); wassert(actual(station.ident.is_missing()).istrue()); wassert(actual(cur).station_keys_match(oldf.stations["metar"].station)); have_metar = true; } } wassert(actual(have_synop).istrue()); wassert(actual(have_metar).istrue()); break; } default: error_unimplemented::throwf("cannot run this test on a database of format %d", (int)DB::format); } wassert(actual(cur->remaining()) == 0); wassert(actual(cur->next()).isfalse()); }); this->add_method("query_best", [](Fixture& f) { // Test querybest OldDballeTestDataSet oldf; wassert(f.populate(oldf)); //if (db.server_type == ORACLE || db.server_type == POSTGRES) // return; // Prepare a query core::Query query; query.latrange = LatRange(1000000, LatRange::IMAX); query.query = "best"; // Make the query auto cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 4); // There should be four items wassert(actual(cur->next()).istrue()); DBStation station = cur->get_station(); wassert(actual(station.coords) == Coords(12.34560, 76.54320)); wassert(actual(station.ident.is_missing()).istrue()); wassert(actual(station.report) == "synop"); wassert(actual(cur->get_level()) == Level(10, 11, 15, 22)); wassert(actual(cur->get_trange()) == Trange(20, 111, 122)); wassert(actual(cur->get_varcode()) == WR_VAR(0, 1, 11)); wassert(actual(cur->get_var().code()) == WR_VAR(0, 1, 11)); wassert(actual(cur->remaining()) == 3); wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 2); wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 0); // Now there should not be anything anymore wassert(actual(cur->next()).isfalse()); }); this->add_method("delete", [](Fixture& f) { // Test deletion OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // 4 items to begin with core::Query query; auto cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 4); cur->discard(); query.clear(); query.dtrange = DatetimeRange(Datetime(1945, 4, 25, 8, 10), Datetime()); f.tr->remove_data(query); // 2 remaining after remove query.clear(); cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 2); cur->discard(); // Did it remove the right ones? query.clear(); query.latrange = LatRange(10.0, LatRange::DMAX); cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 2); wassert(actual(cur->next()).istrue()); wassert(actual(cur).data_context_matches(oldf.data["synop"])); Varcode last_code = 0; for (unsigned i = 0; i < 2; ++i) { // Check that varcodes do not repeat if (last_code != 0) wassert(actual(cur->get_varcode()) != last_code); last_code = cur->get_varcode(); switch (last_code) { case WR_VAR(0, 1, 11): case WR_VAR(0, 1, 12): wassert(actual(cur).data_var_matches(oldf.data["synop"], last_code)); break; default: throw TestFailed("got a varcode that we did not ask for: " + varcode_format(last_code)); } if (i == 0) { /* The item should have two data in it */ wassert(actual(cur->next()).istrue()); } else { wassert(actual(cur->next()).isfalse()); } } }); this->add_method("delete_notfound", [](Fixture& f) { // Test deletion OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // 4 items to begin with core::Query query; auto cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 4); cur->discard(); // Try to remove using a query that matches none query.attr_filter = "B33007<50"; f.tr->remove_data(query); // Verify that nothing has been deleted query.clear(); cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 4); cur->discard(); }); this->add_method("query_datetime", [](Fixture& f) { // Test datetime queries /* Prepare test data */ core::Data base; base.station.coords = Coords(12.0, 48.0); base.station.report = "synop"; base.level = Level(1, 0, 1, 0); base.trange = Trange(1, 0, 0); base.values.set("B01012", 500); #define WANTRESULT(querystr, ab) do { \ auto cur = f.tr->query_data(*dballe::tests::query_from_string(querystr)); \ wassert(actual(cur->remaining()) == 1); \ wassert(actual(cur->next()).istrue()); \ wassert(actual(cur->remaining()) == 0); \ wassert(actual_varcode(cur->get_varcode()) == WR_VAR(0, 1, 12)); \ wassert(actual(cur->get_datetime()) == cur->get_datetime()); \ cur->discard(); \ } while(0) core::Data a, b; impl::DBInsertOptions disallow; disallow.can_replace = disallow.can_add_stations = false; /* Year */ f.tr->remove_all(); a = base; a.datetime = Datetime(2005); f.tr->insert_data(a); b = base; b.datetime = Datetime(2006); f.tr->insert_data(b, disallow); WANTRESULT("yearmin=2006", b); WANTRESULT("yearmax=2005", a); WANTRESULT("year=2006", b); /* Month */ f.tr->remove_all(); a = base; a.datetime = Datetime(2006, 4); f.tr->insert_data(a); b = base; b.datetime = Datetime(2006, 5); f.tr->insert_data(b, disallow); WANTRESULT("year=2006, monthmin=5", b); WANTRESULT("year=2006, monthmax=4", a); WANTRESULT("year=2006, month=5", b); /* Day */ f.tr->remove_all(); a = base; a.datetime = Datetime(2006, 5, 2); f.tr->insert_data(a); b = base; b.datetime = Datetime(2006, 5, 3); f.tr->insert_data(b, disallow); WANTRESULT("year=2006, month=5, daymin=3", b); WANTRESULT("year=2006, month=5, daymax=2", a); WANTRESULT("year=2006, month=5, day=3", b); /* Hour */ f.tr->remove_all(); a = base; a.datetime = Datetime(2006, 5, 3, 12); f.tr->insert_data(a); b = base; b.datetime = Datetime(2006, 5, 3, 13); f.tr->insert_data(b, disallow); WANTRESULT("year=2006, month=5, day=3, hourmin=13", b); WANTRESULT("year=2006, month=5, day=3, hourmax=12", a); WANTRESULT("year=2006, month=5, day=3, hour=13", b); /* Minute */ f.tr->remove_all(); a = base; a.datetime = Datetime(2006, 5, 3, 12, 29); f.tr->insert_data(a); b = base; b.datetime = Datetime(2006, 5, 3, 12, 30); f.tr->insert_data(b, disallow); WANTRESULT("year=2006, month=5, day=3, hour=12, minumin=30", b); WANTRESULT("year=2006, month=5, day=3, hour=12, minumax=29", a); WANTRESULT("year=2006, month=5, day=3, hour=12, min=30", b); }); this->add_method("attrs", [](Fixture& f) { // Test QC OldDballeTestDataSet oldf; wassert(f.populate(oldf)); core::Query query; query.latrange.set(1000000, LatRange::IMAX); auto cur = f.tr->query_data(query); // Move the cursor to B01011 int context_id = 0; bool found = false; while (cur->next()) { if (cur->get_varcode() == WR_VAR(0, 1, 11)) { context_id = dynamic_cast(cur.get())->attr_reference_id(); cur->discard(); found = true; break; } } wassert(actual(found).istrue()); // Insert new attributes about this report Values qc; qc.set("B33002", 2); qc.set("B33003", 5); qc.set("B33005", 33); f.tr->attr_insert_data(context_id, qc); // Query back the data qc.clear(); wassert(actual(run_attr_query_data(f.tr, context_id, qc)) == 3); const auto* attr = qc.maybe_var("B33002"); wassert(actual(attr).istrue()); wassert(actual(attr->enqi()) == 2); attr = qc.maybe_var("B33003"); wassert(actual(attr).istrue()); wassert(actual(attr->enqi()) == 5); attr = qc.maybe_var("B33005"); wassert(actual(attr).istrue()); wassert(actual(attr->enqi()) == 33); // Delete a couple of items vector codes; codes.push_back(WR_VAR(0, 33, 2)); codes.push_back(WR_VAR(0, 33, 5)); f.tr->attr_remove_data(context_id, codes); // Deleting non-existing items should not fail. Also try creating a // query with just one item codes.clear(); codes.push_back(WR_VAR(0, 33, 2)); f.tr->attr_remove_data(context_id, codes); /* Query back the data */ qc.clear(); wassert(actual(run_attr_query_data(f.tr, context_id, qc)) == 1); wassert(actual(qc.maybe_var("B33002")).isfalse()); wassert(actual(qc.maybe_var("B33005")).isfalse()); attr = qc.maybe_var("B33003"); wassert(actual(attr).istrue()); wassert(actual(attr->enqi()) == 5); /*dba_error_remove_callback(DBA_ERR_NONE, crash, 0);*/ }); this->add_method("query_station", [](Fixture& f) { // Test station queries OldDballeTestDataSet oldf; wassert(f.populate(oldf)); auto cur = f.tr->query_stations(*query_from_string("rep_memo=synop")); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); wassert(actual(cur->next()).isfalse()); }); this->add_method("attrs1", [](Fixture& f) { // Test attributes OldDballeTestDataSet oldf; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; // Insert a data record f.tr->insert_data(oldf.data["synop"], opts); Values qc; qc.set("B01007", 1); qc.set("B02048", 2); qc.set("B05040", 3); qc.set("B05041", 4); qc.set("B05043", 5); qc.set("B33032", 6); qc.set("B07024", 7); qc.set("B05021", 8); qc.set("B07025", 9); qc.set("B05022", 10); f.tr->attr_insert_data(oldf.data["synop"].values.value(WR_VAR(0, 1, 11)).data_id, qc); // Query back the B01011 variable to read the attr reference id auto cur = f.tr->query_data(*query_from_string("var=B01011")); wassert(actual(cur->remaining()) == 1); cur->next(); int attr_id = dynamic_cast(cur.get())->attr_reference_id(); cur->discard(); qc.clear(); wassert(actual(run_attr_query_data(f.tr, attr_id, qc)) == 10); // Check that all the attributes come out wassert(actual(qc.size()) == 10); wassert(actual_varcode(qc.var("B01007").code()) == WR_VAR(0, 1, 7)); wassert(actual(qc.var("B01007")) == 1); wassert(actual_varcode(qc.var("B02048").code()) == WR_VAR(0, 2, 48)); wassert(actual(qc.var("B02048")) == 2); wassert(actual_varcode(qc.var("B05021").code()) == WR_VAR(0, 5, 21)); wassert(actual(qc.var("B05021")) == 8); wassert(actual_varcode(qc.var("B05022").code()) == WR_VAR(0, 5, 22)); wassert(actual(qc.var("B05022")) == 10); wassert(actual_varcode(qc.var("B05040").code()) == WR_VAR(0, 5, 40)); wassert(actual(qc.var("B05040")) == 3); wassert(actual_varcode(qc.var("B05041").code()) == WR_VAR(0, 5, 41)); wassert(actual(qc.var("B05041")) == 4); wassert(actual_varcode(qc.var("B05043").code()) == WR_VAR(0, 5, 43)); wassert(actual(qc.var("B05043")) == 5); wassert(actual_varcode(qc.var("B07024").code()) == WR_VAR(0, 7, 24)); wassert(actual(qc.var("B07024")) == 7); wassert(actual_varcode(qc.var("B07025").code()) == WR_VAR(0, 7, 25)); wassert(actual(qc.var("B07025")) == 9); wassert(actual_varcode(qc.var("B33032").code()) == WR_VAR(0, 33, 32)); wassert(actual(qc.var("B33032")) == 6); }); this->add_method("longitude_wrap", [](Fixture& f) { // Test longitude wrapping around OldDballeTestDataSet oldf; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; // Insert a data record f.tr->insert_data(oldf.data["synop"], opts); auto cur = f.tr->query_data(*query_from_string("latmin=10.0, latmax=15.0, lonmin=70.0, lonmax=-160.0")); wassert(actual(cur->remaining()) == 2); cur->discard(); }); this->add_method("query_ana_filter", [](Fixture& f) { // Test numeric comparisons in ana_filter OldDballeTestDataSet oldf; wassert(f.populate(oldf)); auto cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011")); wassert(actual(cur->remaining()) == 1); // Move the cursor to B01011 cur->next(); int context_id = dynamic_cast(cur.get())->attr_reference_id(); cur->discard(); // Insert new attributes about this report Values qc; qc.set("B01001", 50); qc.set("B01008", "50"); f.tr->attr_insert_data(context_id, qc); // Try queries filtered by numeric attributes cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01001=50")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01001<=50")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01001<51")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01001<8")); wassert(actual(cur->remaining()) == 0); cur->discard(); // Try queries filtered by string attributes cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01008=50")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01008<=50")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01008<8")); wassert(actual(cur->remaining()) == 1); cur->discard(); cur = f.tr->query_data(*query_from_string("rep_memo=metar, var=B01011, attr_filter=B01008<100")); wassert(actual(cur->remaining()) == 0); cur->discard(); }); this->add_method("query_station_best", [](Fixture& f) { // Reproduce a querybest scenario which produced invalid SQL OldDballeTestDataSet oldf; wassert(f.populate(oldf)); core::Query q; q.dtrange = DatetimeRange(Datetime(1000, 1, 1, 0, 0, 0), Datetime(1000, 1, 1, 0, 0, 0)); q.query = "best"; auto cur = f.tr->query_data(q); while (cur->next()) { } }); this->add_method("query_best_bug1", [](Fixture& f) { // Reproduce a querybest scenario which produced always the same data record // Import lots const char** files = dballe::tests::bufr_files; auto opts = DBImportOptions::create(); opts->import_attributes = true; opts->update_station = true; opts->overwrite = true; for (int i = 0; files[i] != NULL; i++) { impl::Messages inmsgs = read_msgs(files[i], Encoding::BUFR); wassert(f.tr->import_message(*inmsgs[0], *opts)); } // Query all with best auto cur = f.tr->query_data(*query_from_string("var=B12101, query=best")); unsigned orig_count = cur->remaining(); unsigned count = 0; int id_data = 0; unsigned id_data_changes = 0; while (cur->next()) { ++count; int attr_reference_id = dynamic_cast(cur.get())->attr_reference_id(); if (attr_reference_id != id_data) { id_data = attr_reference_id; ++id_data_changes; } } wassert(actual(count) > 1); wassert(actual(id_data_changes) == count); wassert(actual(count) == orig_count); }); this->add_method("query_invalid_sql", [](Fixture& f) { // Reproduce a query that generated invalid SQL on V6 OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // All DB f.tr->query_stations(*query_from_string("leveltype1=103, l1=2000")); }); this->add_method("update", [](Fixture& f) { // Test value update OldDballeTestDataSet oldf; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; core::Data dataset = oldf.data["synop"]; f.tr->insert_data(dataset, opts); Values attrs; attrs.set("B33007", 50); f.tr->attr_insert_data(dataset.values.value("B01012").data_id, attrs); core::Query q; q.latrange.set(12.34560, 12.34560); q.lonrange.set(76.54320, 76.54320); q.dtrange = DatetimeRange(Datetime(1945, 4, 25, 8, 0, 0), Datetime(1945, 4, 25, 8, 0, 0)); q.report = "synop"; q.level = Level(10, 11, 15, 22); q.trange = Trange(20, 111, 122); q.varcodes.insert(WR_VAR(0, 1, 12)); // Query the initial value auto cur = f.tr->query_data(q); wassert(actual(cur->remaining()) == 1); cur->next(); int ana_id = cur->get_station().id; wreport::Var var = cur->get_var(); wassert(actual(var.enqi()) == 300); // Query the attributes and check that they are there Values qattrs; wassert(actual(run_attr_query_data(f.tr, dynamic_cast(cur.get())->attr_reference_id(), qattrs)) == 1); wassert(actual(qattrs.var("B33007").enq(MISSING_INT)) == 50); // Update it core::Data update; update.station.id = ana_id; update.station.report = "synop"; update.datetime = q.dtrange.min; update.level = q.level; update.trange = q.trange; update.values.set(var.code(), 200); opts.can_replace = true; opts.can_add_stations = false; wassert(f.tr->insert_data(update, opts)); // Query again cur = f.tr->query_data(q); wassert(actual(cur->remaining()) == 1); cur->next(); var = cur->get_var(); wassert(actual(var.enqi()) == 200); qattrs.clear(); // V7 databases implement the semantics in #44 where updating a // value removes the attributes switch (DB::format) { case Format::V7: wassert(actual(run_attr_query_data(f.tr, dynamic_cast(cur.get())->attr_reference_id(), qattrs)) == 0); break; default: throw TestFailed("Database format " + to_string((int)DB::format) + " not supported"); } }); this->add_method("query_stepbystep", [](Fixture& f) { // Try a query checking all the steps OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // Make the query auto cur = f.tr->query_data(*query_from_string("latmin=10.0")); wassert(actual(cur->remaining()) == 4); wassert(actual(cur->next()).istrue()); // remaining() should decrement wassert(actual(cur->remaining()) == 3); // results should match what was inserted wassert(actual(cur).data_matches(oldf.data["metar"])); // just call to_record now, to check if in the next call old variables are removed wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 2); wassert(actual(cur).data_matches(oldf.data["metar"])); wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 1); wassert(actual(cur).data_matches(oldf.data["synop"])); wassert(actual(cur->next()).istrue()); wassert(actual(cur->remaining()) == 0); wassert(actual(cur).data_matches(oldf.data["synop"])); // Now there should not be anything anymore wassert(actual(cur->remaining()) == 0); wassert(actual(cur->next()).isfalse()); }); this->add_method("insert_stationinfo_twice", [](Fixture& f) { // Test double insert of station info NavileDataSet ds; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; //wassert(actual(f.db).empty()); f.tr->insert_station_data(ds.stations["synop"], opts); f.tr->insert_station_data(ds.stations["synop"], opts); // Query station data and ensure there is only one info (height) core::Query query; auto cur = f.tr->query_station_data(query); wassert(actual(cur->remaining()) == 1); cur->next(); wassert(actual(cur).station_keys_match(ds.stations["synop"].station)); wassert(actual(cur).data_var_matches(ds.stations["synop"].values)); }); this->add_method("insert_stationinfo_twice1", [](Fixture& f) { // Test double insert of station info NavileDataSet ds; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; ds.stations["metar"] = ds.stations["synop"]; ds.stations["metar"].station.report = "metar"; f.tr->insert_station_data(ds.stations["synop"], opts); f.tr->insert_station_data(ds.stations["metar"], opts); // Query station data and ensure there is only one info (height) core::Query query; auto cur = f.tr->query_station_data(query); wassert(actual(cur->remaining()) == 2); // Ensure that the network info is preserved // Use a sorted vector because while all DBs group by report, not all DBs // sort by report name. vector reports; while (cur->next()) reports.push_back(cur->get_station().report); std::sort(reports.begin(), reports.end()); wassert(actual(reports[0]) == "metar"); wassert(actual(reports[1]) == "synop"); }); this->add_method("insert_undefined_level2", [](Fixture& f) { // Test handling of values with undefined leveltype2 and l2 OldDballeTestDataSet oldf; impl::DBInsertOptions opts; opts.can_replace = opts.can_add_stations = true; // Insert with undef leveltype2 and l2 core::Data dataset; dataset.station = oldf.data["synop"].station; dataset.datetime = oldf.data["synop"].datetime; dataset.level = Level(44, 55); dataset.trange = Trange(20); dataset.values.set("B01012", 300); f.tr->insert_data(dataset, opts); // Query it back auto cur = f.tr->query_data(*query_from_string("leveltype1=44, l1=55")); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); wassert(actual(cur->get_level()) == Level(44, 55)); wassert(actual(cur->get_trange()) == Trange(20)); wassert(actual(cur->next()).isfalse()); }); this->add_method("query_undefined_level2", [](Fixture& f) { // Test handling of values with undefined leveltype2 and l2 OldDballeTestDataSet oldf; wassert(f.populate(oldf)); // Query with undef leveltype2 and l2 auto cur = f.tr->query_data(*query_from_string("leveltype1=10, l1=11")); wassert(actual(cur->remaining()) == 4); cur->discard(); }); this->add_method("query_bad_attrfilter", [](Fixture& f) { // Query with an incorrect attr_filter OldDballeTestDataSet oldf; wassert(f.populate(oldf)); try { f.tr->query_data(*query_from_string("attr_filter=B12001")); } catch (error_consistency& e) { wassert(actual(e.what()).matches("B12001 is not a valid filter|cannot find any operator in filter 'B12001'")); } }); this->add_method("query_best_priomax", [](Fixture& f) { // Test querying priomax together with query=best // Prepare the common parts of some data core::Data insert; insert.station.coords = Coords(1.0, 1.0); insert.level = Level(1, 0); insert.trange = Trange(254, 0, 0); insert.datetime = Datetime(2009, 11, 11, 0, 0, 0); // 1,synop,synop,101,oss,0 // 2,metar,metar,81,oss,0 // 3,temp,sounding,98,oss,2 // 4,pilot,wind profile,80,oss,2 // 9,buoy,buoy,50,oss,31 // 10,ship,synop ship,99,oss,1 // 11,tempship,temp ship,100,oss,2 // 12,airep,airep,82,oss,4 // 13,amdar,amdar,97,oss,4 // 14,acars,acars,96,oss,4 // 42,pollution,pollution,199,oss,8 // 200,satellite,NOAA satellites,41,oss,255 // 255,generic,generic data,1000,?,255 static const char* rep_memos[] = { "synop", "metar", "temp", "pilot", "buoy", "ship", "tempship", "airep", "amdar", "acars", "pollution", "satellite", "generic", NULL }; for (const char** i = rep_memos; *i; ++i) { insert.clear_ids(); insert.station.report = *i; insert.values.set("B12101", (int)(i - rep_memos)); f.tr->insert_data(insert); } // Query with querybest only { core::Query query; query.query = "best"; query.dtrange = DatetimeRange(Datetime(2009, 11, 11, 0, 0, 0), Datetime(2009, 11, 11, 0, 0, 0)); query.varcodes.insert(WR_VAR(0, 12, 101)); auto cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); wassert(actual(cur->get_station().report) == "generic"); cur->discard(); } // Query with querybest and priomax { core::Query query; query.priomax = 100; query.query = "best"; query.dtrange = DatetimeRange(Datetime(2009, 11, 11, 0, 0, 0), Datetime(2009, 11, 11, 0, 0, 0)); query.varcodes.insert(WR_VAR(0, 12, 101)); auto cur = f.tr->query_data(query); wassert(actual(cur->remaining()) == 1); wassert(actual(cur->next()).istrue()); wassert(actual(cur->get_station().report) == "tempship"); cur->discard(); } }); this->add_method("query_repmemo_in_results", [](Fixture& f) { // Ensure that rep_memo is set in the results OldDballeTestDataSet oldf; wassert(f.populate(oldf)); auto cur = f.tr->query_data(core::Query()); while (cur->next()) wassert_false(cur->get_station().report.empty()); }); } template void CommitTests::register_tests() { this->add_method("fd_leaks", [](Fixture& f) { // Test connect leaks core::Data vals; // Set station data vals.station.coords = Coords(12.34560, 76.54320); vals.station.report = "synop"; vals.values.set("B07030", 42.0); // Height impl::DBInsertOptions opts; opts.can_replace = true; opts.can_add_stations = true; // Assume a max open file limit of 1100 for (unsigned i = 0; i < 1100; ++i) { auto db = DB::create_db(f.backend, false); vals.clear_ids(); wassert(db->insert_station_data(vals, opts)); } }); } } dballe-8.6/dballe/db/summary.cc0000644000175000017500000003634613554564112013371 00000000000000#define _DBALLE_LIBRARY_CODE #include "summary.h" #include "dballe/core/var.h" #include "dballe/core/query.h" #include "dballe/core/json.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include #include #include using namespace std; using namespace dballe; namespace dballe { namespace db { namespace summary { #if 0 std::ostream& operator<<(std::ostream& out, const Entry& e) { return out << "s:" << e.station << " l:" << e.level << " t:" << e.trange << " v:" << e.varcode << " d:" << e.dtrange << " c:" << e.count; } #endif void VarEntry::to_json(core::JSONWriter& writer) const { writer.start_mapping(); writer.add("l", var.level); writer.add("t", var.trange); writer.add("v", var.varcode); writer.add("d", dtrange); writer.add("c", count); writer.end_mapping(); } VarEntry VarEntry::from_json(core::json::Stream& in) { VarEntry res; in.parse_object([&](const std::string& key) { if (key == "l") res.var.level = in.parse_level(); else if (key == "t") res.var.trange = in.parse_trange(); else if (key == "v") res.var.varcode = in.parse_unsigned(); else if (key == "d") res.dtrange = in.parse_datetimerange(); else if (key == "c") res.count = in.parse_unsigned(); else throw core::JSONParseException("unsupported key \"" + key + "\" for summary::VarEntry"); }); return res; } void VarEntry::dump(FILE* out) const { char buf[7]; format_code(var.varcode, buf); fprintf(out, " Level: "); var.level.print(out); fprintf(out, " Trange: "); var.trange.print(out); fprintf(out, " Varcode: %s\n", buf); fprintf(out, " Datetime range: "); dtrange.min.print_iso8601(out, 'T', " to "); dtrange.max.print_iso8601(out); fprintf(out, " Count: %zd\n", count); } template void StationEntry::add(const VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count) { iterator i = find(vd); if (i != end()) i->merge(dtrange, count); else SmallSet::add(VarEntry(vd, dtrange, count)); } template template void StationEntry::add(const StationEntry& entries) { for (const auto& entry: entries) add(entry.var, entry.dtrange, entry.count); } template void StationEntry::add_filtered(const StationEntry& entries, const dballe::Query& query) { const core::Query& q = core::Query::downcast(query); DatetimeRange wanted_dtrange = q.get_datetimerange(); for (const auto& entry: entries) { if (!q.level.is_missing() && q.level != entry.var.level) continue; if (!q.trange.is_missing() && q.trange != entry.var.trange) continue; if (!q.varcodes.empty() && q.varcodes.find(entry.var.varcode) == q.varcodes.end()) continue; if (!wanted_dtrange.contains(entry.dtrange)) continue; add(entry.var, entry.dtrange, entry.count); } } template void StationEntry::to_json(core::JSONWriter& writer) const { writer.start_mapping(); writer.add("s"); writer.add(station); writer.add("v"); writer.start_list(); for (const auto entry: *this) entry.to_json(writer); writer.end_list(); writer.end_mapping(); } template StationEntry StationEntry::from_json(core::json::Stream& in) { StationEntry res; in.parse_object([&](const std::string& key) { if (key == "s") res.station = in.parse(); else if (key == "v") in.parse_array([&]{ res.add(VarEntry::from_json(in)); }); else throw core::JSONParseException("unsupported key \"" + key + "\" for summary::StationEntry"); }); return res; } template void StationEntry::dump(FILE* out) const { fprintf(out, " Station: "); station.print(out); fprintf(out, " Vars:\n"); for (const auto& entry: *this) entry.dump(out); } template void StationEntries::add(const Station& station, const VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count) { iterator cur = this->find(station); if (cur != end()) cur->add(vd, dtrange, count); else Parent::add(StationEntry(station, vd, dtrange, count)); } template void StationEntries::add(const StationEntries& entries) { for (const auto& entry: entries) add(entry); } template void StationEntries::add(const StationEntry& entry) { iterator cur = this->find(entry.station); if (cur != end()) cur->add(entry); else Parent::add(entry); } namespace { Station convert_station(const DBStation& station) { Station res(station); return res; } DBStation convert_station(const Station& station) { DBStation res; res.report = station.report; res.coords = station.coords; res.ident = station.ident; return res; } } template template void StationEntries::add(const StationEntries& entries) { for (const auto& entry: entries) { Station station = convert_station(entry.station); iterator cur = this->find(station); if (cur != end()) cur->add(entry); else Parent::add(StationEntry(station, entry)); } } namespace { struct StationFilterBase { const core::Query& q; bool has_flt_rep_memo; bool has_flt_ident; bool has_flt_area; bool has_flt_station; StationFilterBase(const dballe::Query& query) : q(core::Query::downcast(query)) { // Scan the filter building a todo list of things to match // If there is any filtering on the station, build a whitelist of matching stations has_flt_rep_memo = !q.report.empty(); has_flt_ident = !q.ident.is_missing(); has_flt_area = !q.latrange.is_missing() || !q.lonrange.is_missing(); has_flt_station = has_flt_rep_memo || has_flt_area || has_flt_ident; } template bool matches_station(const Station& station) { if (has_flt_area) { if (!q.latrange.contains(station.coords.lat) || !q.lonrange.contains(station.coords.lon)) return false; } if (has_flt_rep_memo && q.report != station.report) return false; if (has_flt_ident && q.ident != station.ident) return false; return true; } }; template struct StationFilter; template<> struct StationFilter : public StationFilterBase { using StationFilterBase::StationFilterBase; bool matches_station(const Station& station) { return StationFilterBase::matches_station(station); } }; template<> struct StationFilter : public StationFilterBase { StationFilter(const dballe::Query& query) : StationFilterBase(query) { has_flt_station |= (q.ana_id != MISSING_INT); } bool matches_station(const DBStation& station) { if (q.ana_id != MISSING_INT and station.id != q.ana_id) return false; return StationFilterBase::matches_station(station); } }; } template void StationEntries::add_filtered(const StationEntries& entries, const dballe::Query& query) { StationFilter filter(query); if (filter.has_flt_station) { for (auto entry: entries) { if (!filter.matches_station(entry.station)) continue; iterator cur = this->find(entry.station); if (cur != end()) cur->add_filtered(entry, query); else Parent::add(StationEntry(entry, query)); } } else { for (auto entry: entries) Parent::add(StationEntry(entry, query)); } } template Cursor::Cursor(const summary::StationEntries& entries, const Query& query) { const core::Query& q = core::Query::downcast(query); summary::StationFilter filter(query); DatetimeRange wanted_dtrange = q.get_datetimerange(); for (const auto& station_entry: entries) { if (filter.has_flt_station && !filter.matches_station(station_entry.station)) continue; for (const auto& var_entry: station_entry) { if (!q.level.is_missing() && q.level != var_entry.var.level) continue; if (!q.trange.is_missing() && q.trange != var_entry.var.trange) continue; if (!q.varcodes.empty() && q.varcodes.find(var_entry.var.varcode) == q.varcodes.end()) continue; if (!wanted_dtrange.contains(var_entry.dtrange)) continue; results.emplace_back(station_entry, var_entry); } } } template class Cursor; template class Cursor; } template BaseSummary::BaseSummary() { } template std::unique_ptr BaseSummary::query_summary(const Query& query) const { return std::unique_ptr(new summary::Cursor(entries, query)); } template void BaseSummary::recompute_summaries() const { bool first = true; for (const auto& station_entry: entries) { m_reports.add(station_entry.station.report); for (const auto& var_entry: station_entry) { m_levels.add(var_entry.var.level); m_tranges.add(var_entry.var.trange); m_varcodes.add(var_entry.var.varcode); if (first) { first = false; dtrange = var_entry.dtrange; count = var_entry.count; } else { dtrange.merge(var_entry.dtrange); count += var_entry.count; } } } if (first) { dtrange = DatetimeRange(); count = 0; } dirty = false; } template void BaseSummary::add(const Station& station, const summary::VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count) { entries.add(station, vd, dtrange, count); dirty = true; } template void BaseSummary::add_cursor(const dballe::CursorSummary& cur) { add(cur.get_station(), summary::VarDesc(cur.get_level(), cur.get_trange(), cur.get_varcode()), cur.get_datetimerange(), cur.get_count()); } template void BaseSummary::add_message(const dballe::Message& message) { const impl::Message& msg = impl::Message::downcast(message); Station station; // Coordinates station.coords = msg.get_coords(); if (station.coords.is_missing()) throw wreport::error_notfound("coordinates not found in message to summarise"); // Report code station.report = msg.get_report(); // Station identifier station.ident = msg.get_ident(); // Datetime Datetime dt = msg.get_datetime(); DatetimeRange dtrange(dt, dt); // TODO: obtain the StationEntry only once, and add the rest to it, to // avoid looking it up for each variable // Station variables summary::VarDesc vd_ana; vd_ana.level = Level(); vd_ana.trange = Trange(); for (const auto& val: msg.station_data) { vd_ana.varcode = val->code(); add(station, vd_ana, dtrange, 1); } // Variables for (const auto& ctx: msg.data) { summary::VarDesc vd(ctx.level, ctx.trange, 0); for (const auto& val: ctx.values) { if (not val->isset()) continue; vd.varcode = val->code(); add(station, vd, dtrange, 1); } } } template void BaseSummary::add_messages(const std::vector>& messages) { for (const auto& message: messages) add_message(*message); } template void BaseSummary::add_filtered(const BaseSummary& summary, const dballe::Query& query) { entries.add_filtered(summary.entries, query); dirty = true; } template template void BaseSummary::add_summary(const BaseSummary& summary) { entries.add(summary.stations()); dirty = true; } #if 0 void Summary::merge_entries() { if (entries.size() < 2) return; std::sort(entries.begin(), entries.end()); auto first = entries.begin(); auto last = entries.end(); auto tail = first; while (++first != last) { if (tail->same_metadata(*first)) tail->merge(*first); else if (++tail != first) *tail = std::move(*first); } ++tail; if (tail != last) entries.erase(tail, last); } #endif template void BaseSummary::to_json(core::JSONWriter& writer) const { writer.start_mapping(); writer.add("e"); writer.start_list(); for (const auto& e: entries) e.to_json(writer); writer.end_list(); writer.end_mapping(); } template void BaseSummary::load_json(core::json::Stream& in) { in.parse_object([&](const std::string& key) { if (key == "e") in.parse_array([&]{ entries.add(summary::StationEntry::from_json(in)); }); else throw core::JSONParseException("unsupported key \"" + key + "\" for summary::Entry"); }); dirty = true; } template void BaseSummary::dump(FILE* out) const { fprintf(out, "Summary:\n"); fprintf(out, "Stations:\n"); for (const auto& entry: entries) entry.dump(out); fprintf(out, "Reports:\n"); for (const auto& val: m_reports) fprintf(out, " - %s\n", val.c_str()); fprintf(out, "Levels:\n"); for (const auto& val: m_levels) { fprintf(out, " - "); val.print(out); } fprintf(out, "Tranges:\n"); for (const auto& val: m_tranges) { fprintf(out, " - "); val.print(out); } fprintf(out, "Varcodes:\n"); for (const auto& val: m_varcodes) { char buf[7]; format_code(val, buf); fprintf(out, " - %s\n", buf); } fprintf(out, "Datetime range: "); dtrange.min.print_iso8601(out, 'T', " to "); dtrange.max.print_iso8601(out); fprintf(out, "Count: %zd\n", count); fprintf(out, "Dirty: %s\n", dirty ? "true" : "false"); } template class BaseSummary; template void BaseSummary::add_summary(const BaseSummary&); template void BaseSummary::add_summary(const BaseSummary&); template class BaseSummary; template void BaseSummary::add_summary(const BaseSummary&); template void BaseSummary::add_summary(const BaseSummary&); } } dballe-8.6/dballe/db/defs.h0000644000175000017500000000042113554564112012440 00000000000000#ifndef DBALLE_DB_DEFS_H #define DBALLE_DB_DEFS_H #include #include #include namespace dballe { namespace db { /** * Structure uesd to pass lists of varcodes */ typedef std::vector AttrList; } } #endif dballe-8.6/dballe/db/db-test.cc0000644000175000017500000000065013554564112013223 00000000000000#include "dballe/msg/msg.h" #include "dballe/db/tests.h" #include "config.h" #include using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("db_db"); void Tests::register_tests() { add_method("empty", []{ }); } } dballe-8.6/dballe/db/db-basic-test.cc0000644000175000017500000004345313554573614014321 00000000000000#include "dballe/msg/msg.h" #include "dballe/db/tests.h" #include "v7/repinfo.h" #include "v7/db.h" #include "v7/transaction.h" #include "config.h" #include using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; template class CommitTests : public FixtureTestCase> { typedef DBFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg2("db_basic_tr_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_basic_tr_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_basic_tr_v7_mysql", "MYSQL"); #endif CommitTests ct2("db_basic_db_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ CommitTests ct4("db_basic_db_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL CommitTests ct6("db_basic_db_v7_mysql", "MYSQL"); #endif template void Tests::register_tests() { this->add_method("repinfo", [](Fixture& f) { // Test repinfo-related functions std::map prios = f.tr->repinfo().get_priorities(); wassert(actual(prios.find("synop") != prios.end()).istrue()); wassert(actual(prios["synop"]) == 101); int added, deleted, updated; f.tr->update_repinfo((string(getenv("DBA_TESTDATA")) + "/test-repinfo1.csv").c_str(), &added, &deleted, &updated); wassert(actual(added) == 3); wassert(actual(deleted) == 11); wassert(actual(updated) == 2); prios = f.tr->repinfo().get_priorities(); wassert(actual(prios.find("fixspnpo") != prios.end()).istrue()); wassert(actual(prios["fixspnpo"]) == 200); }); this->add_method("simple", [](Fixture& f) { // Test remove_all f.tr->remove_all(); std::unique_ptr cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 0); // Check that it is idempotent f.tr->remove_all(); cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 0); // Insert something OldDballeTestDataSet data_set; wassert(f.populate(data_set)); cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 4); f.tr->remove_all(); cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 0); }); this->add_method("stationdata", [](Fixture& f) { // Test adding station data for different networks // Insert two values in two networks core::Data vals; vals.station.coords = Coords(12.077, 44.600); vals.station.report = "synop"; vals.level = Level(103, 2000); vals.trange = Trange::instant(); vals.datetime = Datetime(2014, 1, 1, 0, 0, 0); vals.values.set("B12101", 273.15); f.tr->insert_data(vals); vals.clear_ids(); vals.station.report = "temp"; vals.values.set("B12101", 274.15); f.tr->insert_data(vals); // Insert station names in both networks core::Data svals_camse; svals_camse.station.coords = vals.station.coords; svals_camse.station.report = "synop"; svals_camse.values.set("B01019", "Camse"); f.tr->insert_station_data(svals_camse); core::Data svals_esmac; svals_esmac.station.coords = vals.station.coords; svals_esmac.station.report = "temp"; svals_esmac.values.set("B01019", "Esmac"); f.tr->insert_station_data(svals_esmac); // Query back all the data auto cur = f.tr->query_stations(core::Query()); // Check results switch (DB::format) { case Format::V7: { bool have_temp = false; bool have_synop = false; // For v7 databases, we get one record per (station, network) // combination for (unsigned i = 0; i < 2; ++i) { wassert(actual(cur->next()).istrue()); DBStation station = cur->get_station(); DBValues values = cur->get_values(); if (station.report == "temp") { wassert(actual(station.id) == svals_esmac.station.id); wassert(actual(values.var(WR_VAR(0, 1, 19))) == "Esmac"); have_temp = true; } else if (station.report == "synop") { wassert(actual(station.id) == svals_camse.station.id); wassert(actual(values.var(WR_VAR(0, 1, 19))) == "Camse"); have_synop = true; } } wassert(actual(cur->next()).isfalse()); wassert(actual(have_temp).istrue()); wassert(actual(have_synop).istrue()); break; } default: throw error_unimplemented("testing stations_without_data on unsupported database"); } impl::Messages msgs; auto cursor = f.tr->query_messages(core::Query()); while (cursor->next()) msgs.push_back(cursor->detach_message()); wassert(actual(msgs.size()) == 2); //msgs.print(stderr); wassert(actual(impl::Message::downcast(msgs[0])->get_rep_memo_var()->enqc()) == "synop"); wassert(actual(impl::Message::downcast(msgs[0])->get_st_name_var()->enqc()) == "Camse"); wassert(actual(impl::Message::downcast(msgs[0])->get_temp_2m_var()->enqd()) == 273.15); wassert(actual(impl::Message::downcast(msgs[1])->get_rep_memo_var()->enqc()) == "temp"); wassert(actual(impl::Message::downcast(msgs[1])->get_st_name_var()->enqc()) == "Esmac"); wassert(actual(impl::Message::downcast(msgs[1])->get_temp_2m_var()->enqd()) == 274.15); }); this->add_method("query_ident", [](Fixture& f) { // Insert a mobile station core::Data vals; vals.station.report = "synop"; vals.station.coords = Coords(44.10, 11.50); vals.station.ident = "foo"; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 295.1); f.tr->insert_data(vals); wassert(actual(f.tr).try_station_query("ident=foo", 1)); wassert(actual(f.tr).try_station_query("ident=bar", 0)); wassert(actual(f.tr).try_station_query("mobile=1", 1)); wassert(actual(f.tr).try_station_query("mobile=0", 0)); wassert(actual(f.tr).try_data_query("ident=foo", 1)); wassert(actual(f.tr).try_data_query("ident=bar", 0)); wassert(actual(f.tr).try_data_query("mobile=1", 1)); wassert(actual(f.tr).try_data_query("mobile=0", 0)); }); this->add_method("ident_case", [](Fixture& f) { // Insert a mobile station core::Data vals; vals.station.report = "synop"; vals.station.coords = Coords(44.10, 11.50); vals.station.ident = "TeSt"; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 295.1); f.tr->insert_data(vals); wassert(actual(f.tr).try_station_query("ident=test", 0)); wassert(actual(f.tr).try_station_query("ident=TeSt", 1)); wassert(actual(f.tr).try_station_query("ident=TEST", 0)); }); this->add_method("ident_encoding", [](Fixture& f) { // Insert a mobile station core::Data vals; vals.station.report = "synop"; vals.station.coords = Coords(44.10, 11.50); vals.station.ident = "ðŸ•"; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 295.1); f.tr->insert_data(vals); wassert(actual(f.tr).try_station_query("ident=ðŸ•", 1)); wassert(actual(f.tr).try_station_query("ident=?", 0)); wassert(actual(f.tr).try_station_query("ident=test", 0)); }); this->add_method("missing_repmemo", [](Fixture& f) { // Test querying with a missing rep_memo core::Query query; query.report = "nonexisting"; wassert(actual(f.tr->query_stations(query)->remaining()) == 0); wassert(actual(f.tr->query_station_data(query)->remaining()) == 0); wassert(actual(f.tr->query_data(query)->remaining()) == 0); wassert(actual(f.tr->query_summary(query)->remaining()) == 0); }); this->add_method("update_with_ana_id", [](Fixture& f) { { core::Data vals; vals.station.report = "synop"; vals.station.coords = Coords(44.10, 11.50); vals.station.ident = "foo"; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 295.1); f.tr->insert_data(vals); } core::Query query; auto cur = f.tr->query_stations(query); wassert(actual(cur->remaining()) == 1u); wassert(cur->next()); int ana_id = cur->get_station().id; // Replace by ana_id { core::Data vals; vals.station.id = ana_id; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 296.2); impl::DBInsertOptions opts; opts.can_replace = true; opts.can_add_stations = false; wassert(f.tr->insert_data(vals, opts)); } auto dcur = f.tr->query_data(query); wassert(actual(dcur->remaining()) == 1u); wassert(dcur->next()); auto var = dcur->get_var(); wassert(actual(var.code()) == WR_VAR(0, 12, 101)); wassert(actual(var.enqd()) == 296.2); // Remove by ana_id query.clear(); query.ana_id = ana_id; query.level = Level(1); query.trange = Trange::instant(); wassert(f.tr->remove_data(query)); query.clear(); dcur = f.tr->query_data(query); wassert(actual(dcur->remaining()) == 0u); // Insert by ana_id { core::Data vals; vals.station.id = ana_id; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 296.2); impl::DBInsertOptions opts; opts.can_replace = false; opts.can_add_stations = false; wassert(f.tr->insert_data(vals, opts)); } dcur = f.tr->query_data(query); wassert(actual(dcur->remaining()) == 1u); wassert(dcur->next()); var = dcur->get_var(); wassert(actual(var.code()) == WR_VAR(0, 12, 101)); wassert(actual(var.enqd()) == 296.2); }); this->add_method("discriminate_ana_id", [](Fixture& f) { // Generate two station IDs core::Data vals1; vals1.station.report = "synop"; vals1.station.coords = Coords(44.10, 11.50); vals1.values.set("B01001", 10); f.tr->insert_station_data(vals1); core::Data vals2; vals2.station.report = "metar"; vals2.station.coords = Coords(44.10, 11.50); vals2.values.set("B01001", 11); f.tr->insert_station_data(vals2); wassert(actual(vals1.station.id) != vals2.station.id); // Insert by ana_id on both stations { core::Data vals; vals.station.id = vals1.station.id; vals.values.set("B01002", 101); impl::DBInsertOptions opts; opts.can_replace = false; opts.can_add_stations = false; wassert(f.tr->insert_station_data(vals, opts)); } { core::Data vals; vals.station.id = vals2.station.id; vals.values.set("B01002", 102); impl::DBInsertOptions opts; opts.can_replace = false; opts.can_add_stations = false; wassert(f.tr->insert_station_data(vals, opts)); } core::Query query; auto cur = f.tr->query_station_data(query); wassert(actual(cur->remaining()) == 4u); wassert(cur->next()); wassert(actual(cur->get_station().id) == vals1.station.id); wassert(actual(cur->get_var().code()) == WR_VAR(0, 1, 1)); wassert(actual(cur->get_var().enqd()) == 10); wassert(cur->next()); wassert(actual(cur->get_station().id) == vals1.station.id); wassert(actual(cur->get_var().code()) == WR_VAR(0, 1, 2)); wassert(actual(cur->get_var().enqd()) == 101); wassert(cur->next()); wassert(actual(cur->get_station().id) == vals2.station.id); wassert(actual(cur->get_var().code()) == WR_VAR(0, 1, 1)); wassert(actual(cur->get_var().enqd()) == 11); wassert(cur->next()); wassert(actual(cur->get_station().id) == vals2.station.id); wassert(actual(cur->get_var().code()) == WR_VAR(0, 1, 2)); wassert(actual(cur->get_var().enqd()) == 102); }); this->add_method("foreign_ana_id", [](Fixture& f) { // Insert a station data core::Data vals1; vals1.station.report = "synop"; vals1.station.coords = Coords(44.10, 11.50); vals1.values.set("B01001", 10); wassert(f.tr->insert_station_data(vals1)); // Insert a station data with an ana_id from another datatabase core::Data vals2; vals2.station.id = vals1.station.id + 42; vals2.station.report = "synop"; vals2.station.coords = Coords(44.10, 11.50); vals2.values.set("B01002", 100); wassert(f.tr->insert_station_data(vals2)); // The right ana_id has been picked wassert(actual(vals1.station.id) == vals2.station.id); }); this->add_method("delete_by_var", [](Fixture& f) { // See issue #141 { core::Data vals; vals.station.report = "synop"; vals.station.coords = Coords(44.10, 11.50); vals.station.ident = "foo"; vals.level = Level(1); vals.trange = Trange::instant(); vals.datetime = Datetime(2015, 4, 25, 12, 30, 45); vals.values.set("B12101", 295.1); vals.values.set("B12103", 295.2); f.tr->insert_data(vals); f.tr->insert_station_data(vals); } // Try deleting station data { auto cur = f.tr->query_station_data(core::Query()); wassert(actual(cur->remaining()) == 2u); } { core::Query query; query.varcodes.insert(WR_VAR(0, 12, 101)); f.tr->remove_station_data(query); } { auto cur = f.tr->query_station_data(core::Query()); wassert(actual(cur->remaining()) == 1u); wassert_true(cur->next()); wassert(actual(cur->get_varcode()) == WR_VAR(0, 12, 103)); } // Try deleting normal data { auto cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 2u); } { core::Query query; query.varcodes.insert(WR_VAR(0, 12, 101)); f.tr->remove_data(query); } { auto cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 1u); wassert_true(cur->next()); wassert(actual(cur->get_varcode()) == WR_VAR(0, 12, 103)); } }); } template void CommitTests::register_tests() { // Test simple queries this->add_method("reset", [](Fixture& f) { // Run twice to see if it is idempotent auto& db = *f.db; db.reset(); db.reset(); }); this->add_method("vacuum", [](Fixture& f) { TestDataSet data; data.stations["s1"].station.report = "synop"; data.stations["s1"].station.coords = Coords(12.34560, 76.54320); data.stations["s1"].values.set("B01019", "Station 1"); data.stations["s2"].station.report = "metar"; data.stations["s2"].station.coords = Coords(23.45670, 65.43210); data.stations["s2"].values.set("B01019", "Station 2"); data.data["s1"].station = data.stations["s1"].station; data.data["s1"].level = Level(10, 11, 15, 22); data.data["s1"].trange = Trange(20, 111, 122); data.data["s1"].datetime = Datetime(1945, 4, 25, 8); data.data["s1"].values.set("B01011", "Data 1"); data.data["s2"].station = data.stations["s2"].station; data.data["s2"].level = Level(10, 11, 15, 22); data.data["s2"].trange = Trange(20, 111, 122); data.data["s2"].datetime = Datetime(1945, 4, 25, 8); data.data["s2"].values.set("B01011", "Data 2"); // Insert some data wassert(f.populate_database(data)); // Invoke vacuum auto& db = *f.db; wassert(db.vacuum()); // Stations are still there { core::Query q; auto c = db.query_stations(q); wassert(actual(c->remaining()) == 2); } // Delete all measured values, but not station values { core::Query q; q.ana_id = data.stations["s1"].station.id; db.remove_data(q); } { core::Query q; // Stations are still there before vacuum auto c = db.query_stations(q); wassert(actual(c->remaining()) == 2); } // Invoke vacuum wassert(db.vacuum()); // Station 1 is gone { core::Query q; q.ana_id = data.stations["s1"].station.id; auto c = db.query_stations(q); wassert(actual(c->remaining()) == 0); } // Station 2 is still there with all its data { auto tr = db.transaction(); core::Query q; q.ana_id = data.stations["s2"].station.id; auto c = wcallchecked(tr->query_stations(q)); wassert(actual(c->remaining()) == 1); auto sd = wcallchecked(tr->query_station_data(q)); wassert(actual(sd->remaining()) == 1); auto dd = wcallchecked(tr->query_data(q)); wassert(actual(dd->remaining()) == 1); } }); // Test simple queries this->add_method("wipe", [](Fixture& f) { // We are connected to an empty database // Insert some data core::Data data; data.station.report = "synop"; data.station.coords = Coords(12.34560, 76.54320); data.values.set("B01019", "Station 1"); wassert(f.db->insert_station_data(data)); // Disconnect f.db.reset(); // Reconnect with wipe auto db = DB::create_db(f.backend, true); // Test that is is empty wassert(actual(db->query_station_data(core::Query())->remaining()) == 0u); // Reinsert data.clear_ids(); wassert(db->insert_station_data(data)); // Disconnect db.reset(); // Reconnect without wipe db = DB::create_db(f.backend, false); // Test that the data is there wassert(actual(db->query_station_data(core::Query())->remaining()) == 1u); }); } } dballe-8.6/dballe/db/summary-access.cc0000644000175000017500000004467213554564124014634 00000000000000#include "summary.h" #include using namespace wreport; namespace dballe { namespace db { namespace summary { namespace { inline int get_station_id(const Station& station) { return MISSING_INT; } inline int get_station_id(const DBStation& station) { return station.id; } } /* * Summary */ template void Cursor::enq(impl::Enq& enq) const { const auto key = enq.key; const auto len = enq.len; switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': enq.set_dballe_int(cur->var_entry.var.level.l1); break; case '2': enq.set_dballe_int(cur->var_entry.var.level.l2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'p': switch (key[1]) { case '1': enq.set_dballe_int(cur->var_entry.var.trange.p1); break; case '2': enq.set_dballe_int(cur->var_entry.var.trange.p2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(cur->station_entry.station.coords.lat); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'o': if (key[2] == 'n') { enq.set_lon(cur->station_entry.station.coords.lon); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->var_entry.var.varcode); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(cur->station_entry.station.ident); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'l': if (memcmp(key + 1, "evel", 4) == 0) { enq.set_level(cur->var_entry.var.level); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "ount", 4) == 0) { enq.set_int(cur->var_entry.count); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(cur->station_entry.station.report); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(get_station_id(cur->station_entry.station)); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!cur->station_entry.station.ident.is_missing()); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(cur->station_entry.station.coords); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'd': if (memcmp(key + 1, "aym", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.day); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[5] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.day); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 's': if (memcmp(key + 1, "ecm", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.second); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[5] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.second); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 't': if (memcmp(key + 1, "range", 5) == 0) { enq.set_trange(cur->var_entry.var.trange); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 7: switch (key[0]) { case 's': if (memcmp(key + 1, "tation", 6) == 0) { enq.set_station(cur->station_entry.station); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'y': if (memcmp(key + 1, "earm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'h': if (memcmp(key + 1, "ourm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.hour); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.hour); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "inum", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.minute); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.minute); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { return; } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(cur->station_entry.station.report); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "onthm", 5) == 0) { switch (key[6]) { case 'a': if (key[7] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.month); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[7] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.month); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': enq.set_dballe_int(cur->var_entry.var.level.ltype1); break; case '2': enq.set_dballe_int(cur->var_entry.var.level.ltype2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { enq.set_dballe_int(cur->var_entry.var.trange.pind); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "ontext_id", 9) == 0) { enq.set_int(cur->var_entry.count); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 11: if (memcmp(key + 0, "datetimem", 9) == 0) { switch (key[9]) { case 'a': if (key[10] == 'x') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_datetime(cur->var_entry.dtrange.max.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[10] == 'n') { if (cur->var_entry.dtrange.is_missing()) return; else enq.set_datetime(cur->var_entry.dtrange.min.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } template void Cursor::enq(impl::Enq& enq) const; template void Cursor::enq(impl::Enq& enq) const; } } } dballe-8.6/dballe/db/db-export-test.cc0000644000175000017500000001333013554573614014550 00000000000000#include "dballe/db/tests.h" #include "dballe/db/db.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/msg.h" #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::impl; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct DBData : public TestDataSet { DBData() { data["ds0"].station.coords = Coords(12.34560, 76.54321); data["ds0"].station.report = "synop"; data["ds0"].datetime = Datetime(1945, 4, 25, 8, 0); data["ds0"].level = Level(1, 2, 0, 3); data["ds0"].trange = Trange(4, 5, 6); data["ds0"].values.set("B01012", 500); data["ds1"] = data["ds0"]; data["ds1"].datetime = Datetime(1945, 4, 26, 8, 0); data["ds1"].values.set("B01012", 400); data["ds2"].station.coords = Coords(12.34560, 76.54321); data["ds2"].station.report = "synop"; data["ds2"].station.ident = "ciao"; data["ds2"].datetime = Datetime(1945, 4, 26, 8, 0); data["ds2"].level = Level(1, 2, 0, 3); data["ds2"].trange = Trange(4, 5, 6); data["ds2"].values.set("B01012", 300); data["ds3"] = data["ds2"]; data["ds3"].station.report = "metar"; data["ds3"].values.set("B01012", 200); } }; template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg2("db_export_v7_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg4("db_export_v7_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg6("db_export_v7_mysql", "MYSQL"); #endif template void Tests::register_tests() { this->add_method("export", [](Fixture& f) { // Simple export DBData test_data; wassert(f.populate(test_data)); // Put some data in the database and check that it gets exported properly // Query back the data impl::Messages messages = dballe::tests::messages_from_db(f.tr, core::Query()); wassert(actual(messages.size()) == 4u); int synmsg = 2; int metmsg = 3; if (impl::Message::downcast(messages[2])->type == MessageType::METAR) { // Since the order here is not determined, enforce one std::swap(synmsg, metmsg); } wassert(actual(impl::Message::downcast(messages[0])->type) == MessageType::SYNOP); wassert(actual_var(*messages[0], sc::latitude) == 12.34560); wassert(actual_var(*messages[0], sc::longitude) == 76.54321); wassert(actual(*messages[0]).is_undef(sc::ident)); wassert(actual(messages[0]->get_datetime()) == Datetime(1945, 4, 25, 8, 0, 0)); wassert(actual_var(*messages[0], WR_VAR(0, 1, 12), Level(1, 2, 0, 3), Trange(4, 5, 6)) == 500); wassert(actual(impl::Message::downcast(messages[1])->type) == MessageType::SYNOP); wassert(actual_var(*messages[1], sc::latitude) == 12.34560); wassert(actual_var(*messages[1], sc::longitude) == 76.54321); wassert(actual(*messages[1]).is_undef(sc::ident)); wassert(actual(messages[1]->get_datetime()) == Datetime(1945, 4, 26, 8, 0, 0)); wassert(actual_var(*messages[1], WR_VAR(0, 1, 12), Level(1, 2, 0, 3), Trange(4, 5, 6)) == 400); wassert(actual(impl::Message::downcast(messages[synmsg])->type) == MessageType::SYNOP); wassert(actual_var(*messages[synmsg], sc::latitude) == 12.34560); wassert(actual_var(*messages[synmsg], sc::longitude) == 76.54321); wassert(actual_var(*messages[synmsg], sc::ident), "ciao"); wassert(actual(messages[synmsg]->get_datetime()) == Datetime(1945, 4, 26, 8, 0, 0)); wassert(actual_var(*messages[synmsg], WR_VAR(0, 1, 12), Level(1, 2, 0, 3), Trange(4, 5, 6)) == 300); wassert(actual(impl::Message::downcast(messages[metmsg])->type) == MessageType::METAR); wassert(actual_var(*messages[metmsg], sc::latitude) == 12.34560); wassert(actual_var(*messages[metmsg], sc::longitude) == 76.54321); wassert(actual_var(*messages[metmsg], sc::ident), "ciao"); wassert(actual(messages[metmsg]->get_datetime()) == Datetime(1945, 4, 26, 8, 0, 0)); wassert(actual_var(*messages[metmsg], WR_VAR(0, 1, 12), Level(1, 2, 0, 3), Trange(4, 5, 6)) == 200); }); this->add_method("export", [](Fixture& f) { // Text exporting of extra station information // Import some data in the station extra information context core::Data st; // do not set datetime, level, trange, to insert a station variable st.station.coords = Coords(45.0, 11.0); st.station.report = "synop"; st.values.set("B01001", 10); f.tr->insert_station_data(st); // Import one real datum core::Data dv; dv.station = st.station; dv.datetime = Datetime(2000, 1, 1, 0, 0, 0); dv.level = Level(103, 2000); dv.trange = Trange(254, 0, 0); dv.values.set("B12101", 290.0); f.tr->insert_data(dv); // Query back the data impl::Messages msgs = dballe::tests::messages_from_db(f.tr, core::Query()); wassert(actual(msgs.size()) == 1u); auto msg = impl::Message::downcast(msgs[0]); wassert(actual(msg->type) == MessageType::SYNOP); wassert(actual_var(*msgs[0], sc::latitude) == 45.0); wassert(actual_var(*msgs[0], sc::longitude) == 11.0); wassert(actual(*msgs[0]).is_undef(sc::ident)); wassert(actual_var(*msgs[0], sc::block) == 10); wassert(actual_var(*msgs[0], sc::temp_2m) == 290.0); }); this->add_method("missing_repmemo", [](Fixture& f) { // Text exporting of extra station information core::Query query; query.report = "nonexisting"; impl::Messages msgs = wcallchecked(dballe::tests::messages_from_db(f.tr, query)); wassert(actual(msgs.size()) == 0u); }); } } dballe-8.6/dballe/db/db.h0000644000175000017500000003162513554564112012116 00000000000000#ifndef DBALLE_DB_DB_H #define DBALLE_DB_DB_H #include #include #include #include #include #include #include #include #include #include #include #include /** @file * @ingroup db * * Functions used to connect to DB-All.e and insert, query and delete data. */ namespace dballe { namespace impl { /// DBImportOptions with public constructor and copy, safe to use in dballe code but not accessible from the public API struct DBImportOptions : public dballe::DBImportOptions { DBImportOptions() = default; DBImportOptions(const DBImportOptions& o) = default; DBImportOptions(DBImportOptions&& o) = default; DBImportOptions& operator=(const DBImportOptions&) = default; DBImportOptions& operator=(DBImportOptions&&) = default; }; /// DBInsertOptions with public constructor and copy, safe to use in dballe code but not accessible from the public API struct DBInsertOptions : public dballe::DBInsertOptions { DBInsertOptions() = default; DBInsertOptions(const DBInsertOptions& o) = default; DBInsertOptions(DBInsertOptions&& o) = default; DBInsertOptions& operator=(const DBInsertOptions&) = default; DBInsertOptions& operator=(DBInsertOptions&&) = default; }; } namespace db { /// Format a db::Format value to a string std::string format_format(Format format); /// Parse a formatted db::Format value Format format_parse(const std::string& str); struct CursorStation : public impl::CursorStation { /// Remove this station and all its data virtual void remove() = 0; /** * Iterate the cursor until the end, returning the number of items. * * If dump is a FILE pointer, also dump the cursor values to it */ virtual unsigned test_iterate(FILE* dump=0) = 0; }; struct CursorStationData : public impl::CursorStationData { /// Remove this datum virtual void remove() = 0; /// Get the database that created this cursor virtual std::shared_ptr get_transaction() const = 0; /** * Return an integer value that can be used to refer to the current * variable for attribute access */ virtual int attr_reference_id() const = 0; /** * Query/return the attributes for the current value of this cursor */ virtual void query_attrs(std::function)> dest, bool force_read=false) = 0; /// Insert/update attributes for the current variable virtual void insert_attrs(const Values& attrs); /// Remove attributes for the current variable virtual void remove_attrs(const db::AttrList& attrs); /** * Iterate the cursor until the end, returning the number of items. * * If dump is a FILE pointer, also dump the cursor values to it */ virtual unsigned test_iterate(FILE* dump=0) = 0; }; struct CursorData : public impl::CursorData { /// Remove this datum virtual void remove() = 0; /// Get the database that created this cursor virtual std::shared_ptr get_transaction() const = 0; /** * Return an integer value that can be used to refer to the current * variable for attribute access */ virtual int attr_reference_id() const = 0; /** * Query/return the attributes for the current value of this cursor */ virtual void query_attrs(std::function)> dest, bool force_read=false) = 0; /// Insert/update attributes for the current variable virtual void insert_attrs(const Values& attrs); /// Remove attributes for the current variable virtual void remove_attrs(const db::AttrList& attrs); /** * Iterate the cursor until the end, returning the number of items. * * If dump is a FILE pointer, also dump the cursor values to it */ virtual unsigned test_iterate(FILE* dump=0) = 0; }; struct CursorSummary : public impl::CursorSummary { /// Remove all data summarised by this entry virtual void remove() = 0; /** * Iterate the cursor until the end, returning the number of items. * * If dump is a FILE pointer, also dump the cursor values to it */ virtual unsigned test_iterate(FILE* dump=0) = 0; }; class Transaction : public dballe::Transaction { public: virtual ~Transaction() {} /** * Clear state information cached during the transaction. * * This can be used when, for example, a command that deletes data is * issued that would invalidate ID information cached inside the * transaction. */ virtual void clear_cached_state() = 0; /** * Query attributes on a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param dest * The function that will be called on each resulting attribute */ virtual void attr_query_station(int data_id, std::function)> dest) = 0; /** * Query attributes on a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param dest * The function that will be called on each resulting attribute */ virtual void attr_query_data(int data_id, std::function)> dest) = 0; /** * Insert new attributes on a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * The attributes to be added */ virtual void attr_insert_station(int data_id, const Values& attrs) = 0; /** * Insert new attributes on a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * The attributes to be added */ virtual void attr_insert_data(int data_id, const Values& attrs) = 0; /** * Delete attributes from a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * Array of WMO codes of the attributes to delete. If empty, all attributes * associated to the value will be deleted. */ virtual void attr_remove_station(int data_id, const db::AttrList& attrs) = 0; /** * Delete attributes from a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * Array of WMO codes of the attributes to delete. If empty, all attributes * associated to the value will be deleted. */ virtual void attr_remove_data(int data_id, const db::AttrList& attrs) = 0; /** * Update the repinfo table in the database, with the data found in the given * file. * * @param repinfo_file * The name of the CSV file with the report type information data to load. * The file is in CSV format with 6 columns: report code, mnemonic id, * description, priority, descriptor, table A category. * If repinfo_file is NULL, then the default of /etc/dballe/repinfo.csv is * used. * @retval added * The number of repinfo entries that have been added * @retval deleted * The number of repinfo entries that have been deleted * @retval updated * The number of repinfo entries that have been updated */ virtual void update_repinfo(const char* repinfo_file, int* added, int* deleted, int* updated) = 0; /** * Dump the entire contents of the database to an output stream */ virtual void dump(FILE* out) = 0; }; class DB: public dballe::DB { public: static db::Format get_default_format(); static void set_default_format(db::Format format); /** * Create from a SQLite file pathname * * @param pathname * The pathname to a SQLite file */ static std::shared_ptr connect_from_file(const char* pathname); /** * Create an in-memory database */ static std::shared_ptr connect_memory(); /** * Create a database from an open Connection */ static std::shared_ptr create(std::unique_ptr conn); /** * Return TRUE if the string looks like a DB URL * * @param str * The string to test * @return * true if it looks like a URL, else false */ static bool is_url(const char* str); /// Return the format of this DB virtual db::Format format() const = 0; /** * Remove all our traces from the database, if applicable. * * After this has been called, all other DB methods except for reset() will * fail. */ virtual void disappear() = 0; /** * Reset the database, removing all existing Db-All.e tables and re-creating them * empty. * * @param repinfo_file * The name of the CSV file with the report type information data to load. * The file is in CSV format with 6 columns: report code, mnemonic id, * description, priority, descriptor, table A category. * If repinfo_file is NULL, then the default of /etc/dballe/repinfo.csv is * used. */ virtual void reset(const char* repinfo_file=0) = 0; /** * Same as transaction(), but the resulting transaction will throw an * exception if commit is called. Used for tests. */ virtual std::shared_ptr test_transaction(bool readonly=false) = 0; /** * Perform database cleanup operations. * * Orphan values are currently: * \li context values for which no data exists * \li station values for which no context exists * * Depending on database size, this routine can take a few minutes to execute. */ virtual void vacuum() = 0; /** * Query attributes on a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param dest * The function that will be called on each resulting attribute */ virtual void attr_query_station(int data_id, std::function)>&& dest); /** * Query attributes on a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param dest * The function that will be called on each resulting attribute */ virtual void attr_query_data(int data_id, std::function)>&& dest); /** * Insert new attributes on a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * The attributes to be added */ void attr_insert_station(int data_id, const Values& attrs); /** * Insert new attributes on a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * The attributes to be added */ void attr_insert_data(int data_id, const Values& attrs); /** * Delete attributes from a station value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * Array of WMO codes of the attributes to delete. If empty, all attributes * associated to the value will be deleted. */ void attr_remove_station(int data_id, const db::AttrList& attrs); /** * Delete attributes from a data value * * @param data_id * The id (returned by Cursor::attr_reference_id()) used to refer to the value * @param attrs * Array of WMO codes of the attributes to delete. If empty, all attributes * associated to the value will be deleted. */ void attr_remove_data(int data_id, const db::AttrList& attrs); /** * Dump the entire contents of the database to an output stream */ void dump(FILE* out); /// Print informations about the database to the given output stream virtual void print_info(FILE* out); /// Return the default repinfo file pathname static const char* default_repinfo_file(); /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr db) { db::DB* res = dynamic_cast(db.get()); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of DB"); db.release(); return std::unique_ptr(res); } /// Downcast a shared_ptr pointer inline static std::shared_ptr downcast(std::shared_ptr db) { auto res = std::dynamic_pointer_cast(db); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of DB"); return res; } }; } } #endif dballe-8.6/dballe/db/v7/0000755000175000017500000000000013602152021011747 500000000000000dballe-8.6/dballe/db/v7/internals.h0000644000175000017500000000232213554564112014054 00000000000000#ifndef DBALLE_DB_V7_INTERNALS_H #define DBALLE_DB_V7_INTERNALS_H #include #include namespace dballe { namespace db { namespace v7 { /// Store a list of attributes to be inserted/updated in the database struct AttributeList : public std::vector> { void add(wreport::Varcode code, const char* value) { push_back(std::make_pair(code, value)); } /// Get a value by code, returns nullptr if not found const char* get(wreport::Varcode code) const { for (const_iterator i = begin(); i != end(); ++i) if (i->first == code) return i->second; return nullptr; } /** * Get a value by code, returns nullptr if not found, removes it from the * AttributeList */ const char* pop(wreport::Varcode code) { const char* res = nullptr; for (iterator i = begin(); i != end(); ++i) { if (i->first == code) { res = i->second; i->second = nullptr; break; } } while (!empty() && back().second == nullptr) pop_back(); return res; } }; } } } #endif dballe-8.6/dballe/db/v7/levtr-test.cc0000644000175000017500000000220713554564112014326 00000000000000#include "dballe/db/tests.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/levtr.h" #include "config.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public FixtureTestCase> { using FixtureTestCase::FixtureTestCase; typedef EmptyTransactionFixture Fixture; void register_tests() override; }; Tests test_sqlite("db_v7_levtr_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests test_psql("db_v7_levtr_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests test_mysql("db_v7_levtr_mysql", "MYSQL"); #endif void Tests::register_tests() { add_method("insert", [](Fixture& f) { db::v7::Tracer<> trc; auto& lt = f.tr->levtr(); // Insert a lev_tr auto i = lt.obtain_id(trc, db::v7::LevTrEntry(Level(1, 2, 0, 3), Trange(4, 5, 6))); wassert(actual(i) == 1); // Insert another lev_tr i = lt.obtain_id(trc, db::v7::LevTrEntry(Level(2, 3, 1, 4), Trange(5, 6, 7))); wassert(actual(i) == 2); }); } } dballe-8.6/dballe/db/v7/postgresql/0000755000175000017500000000000013602152021014152 500000000000000dballe-8.6/dballe/db/v7/postgresql/levtr.h0000644000175000017500000000205113554564112015413 00000000000000#ifndef DBALLE_DB_V7_POSTGRESQL_LEVTRV7_H #define DBALLE_DB_V7_POSTGRESQL_LEVTRV7_H #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace postgresql { struct DB; /** * Precompiled queries to manipulate the lev_tr table */ struct PostgreSQLLevTr : public v7::LevTr { protected: dballe::sql::PostgreSQLConnection& conn; void _dump(std::function out) override; public: PostgreSQLLevTr(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn); PostgreSQLLevTr(const LevTr&) = delete; PostgreSQLLevTr(const LevTr&&) = delete; PostgreSQLLevTr& operator=(const PostgreSQLLevTr&) = delete; ~PostgreSQLLevTr(); void prefetch_ids(Tracer<>& trc, const std::set& ids) override; const LevTrEntry* lookup_id(Tracer<>& trc, int id) override; int obtain_id(Tracer<>& trc, const LevTrEntry& desc) override; }; } } } } #endif dballe-8.6/dballe/db/v7/postgresql/station.h0000644000175000017500000000265713554564112015754 00000000000000#ifndef DBALLE_DB_V7_POSTGRESQL_STATION_H #define DBALLE_DB_V7_POSTGRESQL_STATION_H #include #include #include namespace wreport { struct Var; } namespace dballe { namespace db { namespace v7 { namespace postgresql { /** * Precompiled queries to manipulate the station table */ class PostgreSQLStation : public v7::Station { protected: /** * DB connection. */ dballe::sql::PostgreSQLConnection& conn; void _dump(std::function out) override; public: PostgreSQLStation(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn); ~PostgreSQLStation(); PostgreSQLStation(const PostgreSQLStation&) = delete; PostgreSQLStation(const PostgreSQLStation&&) = delete; PostgreSQLStation& operator=(const PostgreSQLStation&) = delete; DBStation lookup(Tracer<>& trc, int id_station) override; int maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) override; int insert_new(Tracer<>& trc, const dballe::DBStation& desc) override; void get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) override; void add_station_vars(Tracer<>& trc, int id_station, DBValues& values) override; void run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function) override; }; } } } } #endif dballe-8.6/dballe/db/v7/postgresql/driver.h0000644000175000017500000000160113554564112015552 00000000000000#ifndef DBALLE_DB_V7_POSTGRESQL_DRIVER_H #define DBALLE_DB_V7_POSTGRESQL_DRIVER_H #include #include namespace dballe { namespace db { namespace v7 { namespace postgresql { struct Driver : public v7::Driver { dballe::sql::PostgreSQLConnection& conn; Driver(dballe::sql::PostgreSQLConnection& conn); virtual ~Driver(); std::unique_ptr create_repinfo(v7::Transaction& tr) override; std::unique_ptr create_station(v7::Transaction& tr) override; std::unique_ptr create_levtr(v7::Transaction& tr) override; std::unique_ptr create_data(v7::Transaction& tr) override; std::unique_ptr create_station_data(v7::Transaction& tr) override; void create_tables_v7() override; void delete_tables_v7() override; void vacuum_v7() override; }; } } } } #endif dballe-8.6/dballe/db/v7/postgresql/repinfo.h0000644000175000017500000000236413554564112015730 00000000000000#ifndef DBALLE_DB_V7_POSTGRESQL_REPINFO_H #define DBALLE_DB_V7_POSTGRESQL_REPINFO_H #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace postgresql { /** * Fast cached access to the repinfo table */ struct PostgreSQLRepinfo : public v7::Repinfo { /** * DB connection. The pointer is assumed always valid during the * lifetime of the object */ dballe::sql::PostgreSQLConnection& conn; PostgreSQLRepinfo(dballe::sql::PostgreSQLConnection& conn); PostgreSQLRepinfo(const PostgreSQLRepinfo&) = delete; PostgreSQLRepinfo(const PostgreSQLRepinfo&&) = delete; virtual ~PostgreSQLRepinfo(); PostgreSQLRepinfo& operator=(const PostgreSQLRepinfo&) = delete; void dump(FILE* out) override; protected: /// Return how many time this ID is used in the database int id_use_count(unsigned id, const char* name) override; void delete_entry(unsigned id) override; void update_entry(const v7::repinfo::Cache& entry) override; void insert_entry(const v7::repinfo::Cache& entry) override; void read_cache() override; void insert_auto_entry(const char* memo) override; }; } } } } #endif dballe-8.6/dballe/db/v7/postgresql/repinfo.cc0000644000175000017500000000702713554564112016067 00000000000000#include "repinfo.h" #include "dballe/db/db.h" #include "dballe/sql/postgresql.h" #include "dballe/sql/querybuf.h" using namespace wreport; using namespace std; using dballe::sql::PostgreSQLConnection; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace postgresql { PostgreSQLRepinfo::PostgreSQLRepinfo(PostgreSQLConnection& conn) : Repinfo(conn), conn(conn) { read_cache(); } PostgreSQLRepinfo::~PostgreSQLRepinfo() { } void PostgreSQLRepinfo::read_cache() { cache.clear(); memo_idx.clear(); auto stm = conn.exec("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); for (unsigned row = 0; row < stm.rowcount(); ++row) { cache_append( stm.get_int4(row, 0), stm.get_string(row, 1), stm.get_string(row, 2), stm.get_int4(row, 3), stm.get_string(row, 4), stm.get_int4(row, 5) ); } // Rebuild the memo index as well rebuild_memo_idx(); } void PostgreSQLRepinfo::insert_auto_entry(const char* memo) { unsigned id = conn.exec_one_row("SELECT MAX(id) FROM repinfo").get_int4(0, 0); int prio = conn.exec_one_row("SELECT MAX(prio) FROM repinfo").get_int4(0, 0); ++id; ++prio; Querybuf query(500); query.appendf(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES (%u, $1::text, $1::text, %d, '-', 255) )", id, prio); conn.exec_no_data(query, memo); } int PostgreSQLRepinfo::id_use_count(unsigned id, const char* name) { Querybuf query(500); query.appendf("SELECT COUNT(1) FROM station WHERE rep=%u", id); return conn.exec_one_row(query).get_int4(0, 0); } void PostgreSQLRepinfo::delete_entry(unsigned id) { conn.exec_no_data("DELETE FROM repinfo WHERE id=$1::int4", (int32_t)id); } void PostgreSQLRepinfo::update_entry(const v7::repinfo::Cache& entry) { conn.exec_no_data(R"( UPDATE repinfo SET memo=$2::text, description=$3::text, prio=$4::int4, descriptor=$5::text, tablea=$6::int4 WHERE id=$1::int4 )", (int32_t)entry.id, entry.new_memo, entry.new_desc, (int32_t)entry.new_prio, entry.new_descriptor, (int32_t)entry.new_tablea); } void PostgreSQLRepinfo::insert_entry(const v7::repinfo::Cache& entry) { conn.exec_no_data(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES ($1::int4, $2::text, $3::text, $4::int4, $5::text, $6::int4) )", (int32_t)entry.id, entry.new_memo, entry.new_desc, (int32_t)entry.new_prio, entry.new_descriptor, (int32_t)entry.new_tablea); } void PostgreSQLRepinfo::dump(FILE* out) { fprintf(out, "dump of table repinfo:\n"); fprintf(out, " id memo description prio desc tablea\n"); int count = 0; auto stm = conn.exec("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); for (unsigned row = 0; row < stm.rowcount(); ++row) { string memo = stm.get_string(row, 1); string desc = stm.get_string(row, 2); string descriptor = stm.get_string(row, 4); fprintf(out, " %4d %s %s %d %s %d\n", stm.get_int4(row, 0), memo.c_str(), desc.c_str(), stm.get_int4(row, 3), descriptor.c_str(), stm.get_int4(row, 5)); ++count; }; fprintf(out, "%d element%s in table repinfo\n", count, count != 1 ? "s" : ""); } } } } } dballe-8.6/dballe/db/v7/postgresql/data.cc0000644000175000017500000004656713554564112015352 00000000000000#include "data.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/batch.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/db/v7/repinfo.h" #include "dballe/sql/postgresql.h" #include "dballe/sql/querybuf.h" #include "dballe/values.h" #include "dballe/core/values.h" #include "dballe/core/varmatch.h" #include #include using namespace wreport; using namespace std; using namespace dballe::sql::postgresql; using dballe::sql::PostgreSQLConnection; using dballe::sql::Querybuf; using dballe::sql::error_postgresql; namespace dballe { namespace db { namespace v7 { namespace postgresql { template class PostgreSQLDataCommon; template class PostgreSQLDataCommon; template PostgreSQLDataCommon::PostgreSQLDataCommon(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn) : Parent(tr), conn(conn) { } template void PostgreSQLDataCommon::read_attrs(Tracer<>& trc, int id_data, std::function)> dest) { if (select_attrs_query_name.empty()) { select_attrs_query_name = Parent::table_name; select_attrs_query_name += "v7_select_attrs"; char query[64]; snprintf(query, 64, "SELECT attrs FROM %s WHERE id=$1::int4", Parent::table_name); conn.prepare(select_attrs_query_name, query); } Tracer<> trc_sel(trc ? trc->trace_select("SELECT attrs FROM … WHERE id=$1::int4") : nullptr); Values::decode( conn.exec_prepared_one_row(select_attrs_query_name, id_data).get_bytea(0, 0), dest); if (trc_sel) trc_sel->add_row(); } template void PostgreSQLDataCommon::write_attrs(Tracer<>& trc, int id_data, const Values& values) { if (write_attrs_query_name.empty()) { write_attrs_query_name = Parent::table_name; write_attrs_query_name += "v7_write_attrs"; char query[64]; snprintf(query, 64, "UPDATE %s SET attrs=$1::bytea WHERE id=$2::int4", Parent::table_name); conn.prepare(write_attrs_query_name, query); } Tracer<> trc_upd(trc ? trc->trace_update("UPDATE … SET attrs=$1::bytea WHERE id=$2::int4", 1) : nullptr); vector encoded = values.encode(); conn.exec_prepared_no_data(write_attrs_query_name, encoded, id_data); } template void PostgreSQLDataCommon::remove_all_attrs(Tracer<>& trc, int id_data) { if (remove_attrs_query_name.empty()) { remove_attrs_query_name = Parent::table_name; remove_attrs_query_name += "v7_remove_attrs"; char query[64]; snprintf(query, 64, "UPDATE %s SET attrs=NULL WHERE id=$1::int4", Parent::table_name); conn.prepare(remove_attrs_query_name, query); } Tracer<> trc_upd(trc ? trc->trace_update("UPDATE … SET attrs=NULL WHERE id=$1::int4", 1) : nullptr); conn.exec_prepared_no_data(remove_attrs_query_name, id_data); } namespace { bool match_attrs(const Varmatch& match, const std::vector& attrs) { bool found = false; Values::decode(attrs, [&](std::unique_ptr var) { if (match(*var)) found = true; }); return found; } } template void PostgreSQLDataCommon::remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) { if (!qb.query.attr_filter.empty()) { // We need to apply attr_filter to all results of the query, so we // iterate the results and delete the matching ones one by one. std::unique_ptr attr_filter = Varmatch::parse(qb.query.attr_filter); if (remove_data_query_name.empty()) { remove_data_query_name = Parent::table_name; remove_data_query_name += "v7_remove_data"; char query[64]; snprintf(query, 64, "DELETE FROM %s WHERE id=$1::int4", Parent::table_name); conn.prepare(remove_data_query_name, query); } Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); Result to_remove; if (qb.bind_in_ident) to_remove = conn.exec(qb.sql_query, qb.bind_in_ident); else to_remove = conn.exec(qb.sql_query); if (trc_sel) trc_sel->add_row(to_remove.rowcount()); trc_sel.done(); for (unsigned row = 0; row < to_remove.rowcount(); ++row) { if (!match_attrs(*attr_filter, to_remove.get_bytea(row, 1))) return; Tracer<> trc_del(trc ? trc->trace_delete(remove_data_query_name, 1) : nullptr); conn.exec_prepared(remove_data_query_name, (int)to_remove.get_int4(row, 0)); } } else { Querybuf dq(512); dq.append("DELETE FROM "); dq.append(Parent::table_name); dq.append(" WHERE id IN ("); dq.append(qb.sql_query); dq.append(")"); Tracer<> trc_del(trc ? trc->trace_delete(dq) : nullptr); if (qb.bind_in_ident) { conn.exec_no_data(dq.c_str(), qb.bind_in_ident); } else { conn.exec_no_data(dq.c_str()); } } } template void PostgreSQLDataCommon::remove_by_id(Tracer<>& trc, int id) { char query[64]; snprintf(query, 64, "DELETE FROM %s WHERE id=%d", Parent::table_name, id); // Iterate all the data_id results, deleting the related data and attributes Tracer<> trc_sel(trc ? trc->trace_delete(query, 1) : nullptr); conn.exec_no_data(query); } template void PostgreSQLDataCommon::update(Tracer<>& trc, std::vector& vars, bool with_attrs) { Querybuf qb(512); unsigned count = 0; if (with_attrs) { qb.append("UPDATE "); qb.append(Parent::table_name); qb.append(" as d SET value=i.value, attrs=i.attrs FROM (values "); qb.start_list(","); for (auto& v: vars) { qb.start_list_item(); qb.append("("); qb.append_int(v.id); qb.append(","); conn.append_escaped(qb, v.var->enqc()); qb.append(","); if (v.var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v.var); conn.append_escaped(qb, enc.buf); } else qb.append("NULL"); qb.append("::bytea)"); ++count; } qb.append(") AS i(id, value, attrs) WHERE d.id = i.id"); } else { qb.append("UPDATE "); qb.append(Parent::table_name); qb.append(" as d SET value=i.value, attrs=NULL FROM (values "); qb.start_list(","); for (auto& v: vars) { qb.start_list_item(); qb.append("("); qb.append_int(v.id); qb.append(","); conn.append_escaped(qb, v.var->enqc()); qb.append(")"); ++count; } qb.append(") AS i(id, value) WHERE d.id = i.id"); } //fprintf(stderr, "Update query: %s\n", dq.c_str()); Tracer<> trc_upd(trc ? trc->trace_update(qb, count) : nullptr); conn.exec_no_data(qb); } PostgreSQLStationData::PostgreSQLStationData(v7::Transaction& tr, PostgreSQLConnection& conn) : PostgreSQLDataCommon(tr, conn) { conn.prepare("station_datav7_select", "SELECT id, code FROM station_data WHERE id_station=$1::int4"); } void PostgreSQLStationData::query(Tracer<>& trc, int id_station, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select("station_datav7_select") : nullptr); Result existing(conn.exec_prepared("station_datav7_select", id_station)); if (trc_sel) trc_sel->add_row(existing.rowcount()); for (unsigned row = 0; row < existing.rowcount(); ++row) { int id = existing.get_int4(row, 0); wreport::Varcode code = (Varcode)existing.get_int4(row, 1); dest(id, code); } } void PostgreSQLStationData::insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); char lead[64]; snprintf(lead, 64, "(DEFAULT,%d,", id_station); Querybuf dq(512); dq.append("INSERT INTO station_data (id, id_station, code, value, attrs) VALUES "); dq.start_list(","); unsigned count = 0; for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; dq.start_list_item(); dq.append(lead); dq.append_int(v->var->code()); dq.append(","); conn.append_escaped(dq, v->var->enqc()); dq.append(","); if (with_attrs && v->var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v->var); conn.append_escaped(dq, enc.buf); } else dq.append("NULL::bytea"); dq.append(")"); ++count; } dq.append(" RETURNING id"); //fprintf(stderr, "Insert query: %s\n", dq.c_str()); // Run the insert query and read back the new IDs Tracer<> trc_ins(trc ? trc->trace_insert(dq, count) : nullptr); Result res(conn.exec(dq)); unsigned row = 0; for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; if (row >= res.rowcount()) break; v->id = res.get_int4(row, 0); ++row; } } void PostgreSQLStationData::run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); using namespace dballe::sql::postgresql; // Start the query asynchronously int res; if (qb.bind_in_ident) { const char* args[1] = { qb.bind_in_ident }; res = PQsendQueryParams(conn, qb.sql_query.c_str(), 1, nullptr, args, nullptr, nullptr, 1); } else { res = PQsendQueryParams(conn, qb.sql_query.c_str(), 0, nullptr, nullptr, nullptr, nullptr, 1); } if (!res) throw error_postgresql(conn, "executing " + qb.sql_query); dballe::DBStation station; conn.run_single_row_mode(qb.sql_query, [&](const Result& res) { if (trc_sel) trc_sel->add_row(res.rowcount()); for (unsigned row = 0; row < res.rowcount(); ++row) { wreport::Varcode code = res.get_int4(row, 5); const char* value = res.get_string(row, 7); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(res.get_bytea(row, 8), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = res.get_int4(row, 0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(res.get_int4(row, 1)); station.coords.lat = res.get_int4(row, 2); station.coords.lon = res.get_int4(row, 3); if (res.is_null(row, 4)) station.ident.clear(); else station.ident = res.get_string(row, 4); } int id_data = res.get_int4(row, 6); dest(station, id_data, move(var)); } }); } void PostgreSQLStationData::dump(FILE* out) { StationDataDumper dumper(out); dumper.print_head(); auto res = conn.exec("SELECT id, id_station, code, value, attrs FROM station_data"); for (unsigned row = 0; row < res.rowcount(); ++row) { const char* val = res.is_null(row, 3) ? nullptr : res.get_string(row, 3); dumper.print_row(res.get_int4(row, 0), res.get_int4(row, 1), res.get_int4(row, 2), val, res.get_bytea(row, 4)); } dumper.print_tail(); } PostgreSQLData::PostgreSQLData(v7::Transaction& tr, PostgreSQLConnection& conn) : PostgreSQLDataCommon(tr, conn) { conn.prepare("datav7_select", "SELECT id, id_levtr, code FROM data WHERE id_station=$1::int4 AND datetime=$2::timestamp"); } void PostgreSQLData::query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select("datav7_select") : nullptr); Result existing(conn.exec_prepared("datav7_select", id_station, datetime)); if (trc_sel) trc_sel->add_row(existing.rowcount()); for (unsigned row = 0; row < existing.rowcount(); ++row) { int id = existing.get_int4(row, 0); int id_levtr = existing.get_int4(row, 1); wreport::Varcode code = (Varcode)existing.get_int4(row, 2); dest(id, id_levtr, code); } } void PostgreSQLData::insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); const Datetime& dt = datetime; char val_lead[64]; snprintf(val_lead, 64, "(DEFAULT,%d,'%04d-%02d-%02d %02d:%02d:%02d',", id_station, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); Querybuf dq(512); dq.append("INSERT INTO data (id, id_station, datetime, id_levtr, code, value, attrs) VALUES "); dq.start_list(","); unsigned count = 0; for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; dq.start_list_item(); dq.append(val_lead); dq.append_int(v->id_levtr); dq.append(","); dq.append_int(v->var->code()); dq.append(","); conn.append_escaped(dq, v->var->enqc()); dq.append(","); if (with_attrs && v->var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v->var); conn.append_escaped(dq, enc.buf); } else dq.append("NULL::bytea"); dq.append(")"); ++count; } dq.append(" RETURNING id"); // fprintf(stderr, "Insert query: %s\n", dq.c_str()); // Run the insert query and read back the new IDs Tracer<> trc_ins(trc ? trc->trace_insert(dq, count) : nullptr); Result res(conn.exec(dq)); unsigned row = 0; for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; if (row >= res.rowcount()) break; v->id = res.get_int4(row, 0); ++row; } } void PostgreSQLData::run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); using namespace dballe::sql::postgresql; // Start the query asynchronously int res; if (qb.bind_in_ident) { const char* args[1] = { qb.bind_in_ident }; res = PQsendQueryParams(conn, qb.sql_query.c_str(), 1, nullptr, args, nullptr, nullptr, 1); } else { res = PQsendQueryParams(conn, qb.sql_query.c_str(), 0, nullptr, nullptr, nullptr, nullptr, 1); } if (!res) throw error_postgresql(conn, "executing " + qb.sql_query); dballe::DBStation station; conn.run_single_row_mode(qb.sql_query, [&](const Result& res) { if (trc_sel) trc_sel->add_row(res.rowcount()); for (unsigned row = 0; row < res.rowcount(); ++row) { wreport::Varcode code = res.get_int4(row, 6); const char* value = res.get_string(row, 9); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(res.get_bytea(row, 10), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = res.get_int4(row, 0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(res.get_int4(row, 1)); station.coords.lat = res.get_int4(row, 2); station.coords.lon = res.get_int4(row, 3); if (res.is_null(row, 4)) station.ident.clear(); else station.ident = res.get_string(row, 4); } int id_levtr = res.get_int4(row, 5); int id_data = res.get_int4(row, 7); Datetime datetime = res.get_timestamp(row, 8); dest(station, id_levtr, datetime, id_data, move(var)); } }); } void PostgreSQLData::run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); using namespace dballe::sql::postgresql; // Start the query asynchronously int res; if (qb.bind_in_ident) { const char* args[1] = { qb.bind_in_ident }; res = PQsendQueryParams(conn, qb.sql_query.c_str(), 1, nullptr, args, nullptr, nullptr, 1); } else { res = PQsendQueryParams(conn, qb.sql_query.c_str(), 0, nullptr, nullptr, nullptr, nullptr, 1); } if (!res) throw error_postgresql(conn, "executing " + qb.sql_query); dballe::DBStation station; conn.run_single_row_mode(qb.sql_query, [&](const Result& res) { if (trc_sel) trc_sel->add_row(res.rowcount()); // fprintf(stderr, "ST %d vi %d did %d d %d sd %d\n", qb.select_station, qb.select_varinfo, qb.select_data_id, qb.select_data, qb.select_summary_details); for (unsigned row = 0; row < res.rowcount(); ++row) { int id_station = res.get_int4(row, 0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(res.get_int4(row, 1)); station.coords.lat = res.get_int4(row, 2); station.coords.lon = res.get_int4(row, 3); if (res.is_null(row, 4)) station.ident.clear(); else station.ident = res.get_string(row, 4); } int id_levtr = res.get_int4(row, 5); wreport::Varcode code = res.get_int4(row, 6); size_t count = 0; DatetimeRange datetime; if (qb.select_summary_details) { count = res.get_int8(row, 7); datetime = DatetimeRange(res.get_timestamp(row, 8), res.get_timestamp(row, 9)); } dest(station, id_levtr, code, datetime, count); } }); } void PostgreSQLData::dump(FILE* out) { DataDumper dumper(out); dumper.print_head(); auto res = conn.exec("SELECT id, id_station, id_levtr, datetime, code, value, attrs FROM data"); for (unsigned row = 0; row < res.rowcount(); ++row) { const char* val = res.is_null(row, 5) ? nullptr : res.get_string(row, 5); dumper.print_row(res.get_int4(row, 0), res.get_int4(row, 1), res.get_int4(row, 2), res.get_timestamp(row, 3), res.get_int4(row, 4), val, res.get_bytea(row, 6)); }; dumper.print_tail(); } } } } } dballe-8.6/dballe/db/v7/postgresql/data.h0000644000175000017500000000623313554564112015176 00000000000000#ifndef DBALLE_DB_V7_POSTGRESQL_DATA_H #define DBALLE_DB_V7_POSTGRESQL_DATA_H #include #include #include namespace dballe { namespace db { namespace v7 { namespace postgresql { struct DB; template class PostgreSQLDataCommon : public Parent { protected: /// DB connection dballe::sql::PostgreSQLConnection& conn; std::string select_attrs_query_name; std::string write_attrs_query_name; std::string remove_attrs_query_name; std::string remove_data_query_name; public: PostgreSQLDataCommon(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn); PostgreSQLDataCommon(const PostgreSQLDataCommon&) = delete; PostgreSQLDataCommon(const PostgreSQLDataCommon&&) = delete; PostgreSQLDataCommon& operator=(const PostgreSQLDataCommon&) = delete; void update(Tracer<>& trc, std::vector& vars, bool with_attrs) override; void read_attrs(Tracer<>& trc, int id_data, std::function)> dest) override; void write_attrs(Tracer<>& trc, int id_data, const Values& values) override; void remove_all_attrs(Tracer<>& trc, int id_data) override; void remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) override; void remove_by_id(Tracer<>& trc, int id) override; }; extern template class PostgreSQLDataCommon; extern template class PostgreSQLDataCommon; class PostgreSQLStationData : public PostgreSQLDataCommon { public: using PostgreSQLDataCommon::PostgreSQLDataCommon; PostgreSQLStationData(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn); void query(Tracer<>& trc, int id_station, std::function dest) override; void insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) override; void run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void dump(FILE* out) override; void clear_cache() override {} }; class PostgreSQLData : public PostgreSQLDataCommon { public: using PostgreSQLDataCommon::PostgreSQLDataCommon; PostgreSQLData(v7::Transaction& tr, dballe::sql::PostgreSQLConnection& conn); void query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) override; void insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) override; void run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function) override; void dump(FILE* out) override; void clear_cache() override {} }; } } } } #endif dballe-8.6/dballe/db/v7/postgresql/levtr.cc0000644000175000017500000001146613554564112015563 00000000000000#include "levtr.h" #include "dballe/core/defs.h" #include "dballe/msg/msg.h" #include "dballe/sql/querybuf.h" #include "dballe/sql/postgresql.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include #include #include using namespace wreport; using namespace std; using dballe::sql::PostgreSQLConnection; namespace dballe { namespace db { namespace v7 { namespace postgresql { namespace { Level to_level(const dballe::sql::postgresql::Result& res, unsigned row, int first_id) { return Level( res.get_int4(row, first_id), res.get_int4(row, first_id + 1), res.get_int4(row, first_id + 2), res.get_int4(row, first_id + 3)); } Trange to_trange(const dballe::sql::postgresql::Result& res, unsigned row, int first_id) { return Trange( res.get_int4(row, first_id), res.get_int4(row, first_id + 1), res.get_int4(row, first_id + 2)); } } PostgreSQLLevTr::PostgreSQLLevTr(v7::Transaction& tr, PostgreSQLConnection& conn) : v7::LevTr(tr), conn(conn) { conn.prepare("v7_levtr_select_id", R"( SELECT id FROM levtr WHERE ltype1=$1::int4 AND l1=$2::int4 AND ltype2=$3::int4 AND l2=$4::int4 AND pind=$5::int4 AND p1=$6::int4 AND p2=$7::int4 )"); conn.prepare("v7_levtr_select_data", "SELECT ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id=$1::int4"); conn.prepare("v7_levtr_insert", R"( INSERT INTO levtr (id, ltype1, l1, ltype2, l2, pind, p1, p2) VALUES (DEFAULT, $1::int4, $2::int4, $3::int4, $4::int4, $5::int4, $6::int4, $7::int4) RETURNING id )"); } PostgreSQLLevTr::~PostgreSQLLevTr() { } void PostgreSQLLevTr::prefetch_ids(Tracer<>& trc, const std::set& ids) { if (ids.empty()) return; sql::Querybuf qb; if (ids.size() < 100) { qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id IN ("); qb.start_list(","); for (auto id: ids) qb.append_listf("%d", id); qb.append(")"); } else qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr"); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec(qb); if (trc_sel) trc_sel->add_row(res.rowcount()); for (unsigned row = 0; row < res.rowcount(); ++row) cache.insert(unique_ptr(new LevTrEntry( res.get_int4(row, 0), to_level(res, row, 1), to_trange(res, row, 5)))); } const LevTrEntry* PostgreSQLLevTr::lookup_id(Tracer<>& trc, int id) { using namespace dballe::sql::postgresql; const LevTrEntry* e = cache.find_entry(id); if (e) return e; Tracer<> trc_sel(trc ? trc->trace_select("v7_levtr_select_data") : nullptr); auto res = conn.exec_prepared("v7_levtr_select_data", id); if (trc_sel) trc_sel->add_row(res.rowcount()); switch (res.rowcount()) { case 0: error_notfound::throwf("levtr with id %d not found in the database", id); case 1: return cache.insert(unique_ptr(new LevTrEntry(id, to_level(res, 0, 0), to_trange(res, 0, 4)))); default: error_consistency::throwf("select levtr data query returned %u results", res.rowcount()); } } int PostgreSQLLevTr::obtain_id(Tracer<>& trc, const LevTrEntry& desc) { using namespace dballe::sql::postgresql; int id = cache.find_id(desc); if (id != MISSING_INT) return id; Tracer<> trc_oid(trc ? trc->trace_select("v7_levtr_select_id") : nullptr); Result res = conn.exec_prepared("v7_levtr_select_id", desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); if (trc_oid) trc_oid->add_row(res.rowcount()); switch (res.rowcount()) { case 0: { trc_oid.done(); trc_oid.reset(trc ? trc->trace_insert("v7_levtr_insert", 1) : nullptr); auto res = conn.exec_prepared_one_row("v7_levtr_insert", desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); id = res.get_int4(0, 0); cache.insert(desc, id); return id; } case 1: { id = res.get_int4(0, 0); cache.insert(desc, id); return id; } default: error_consistency::throwf("select levtr ID query returned %u results", res.rowcount()); } } void PostgreSQLLevTr::_dump(std::function out) { auto res = conn.exec("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr ORDER BY ID"); for (unsigned row = 0; row < res.rowcount(); ++row) out(res.get_int4(row, 0), to_level(res, row, 1), to_trange(res, row, 5)); } } } } } dballe-8.6/dballe/db/v7/postgresql/station.cc0000644000175000017500000001653113554564112016106 00000000000000#include "station.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/repinfo.h" #include "dballe/sql/postgresql.h" #include "dballe/core/var.h" #include "dballe/values.h" #include using namespace wreport; using namespace dballe::db; using namespace std; using dballe::sql::PostgreSQLConnection; namespace dballe { namespace db { namespace v7 { namespace postgresql { PostgreSQLStation::PostgreSQLStation(v7::Transaction& tr, PostgreSQLConnection& conn) : v7::Station(tr), conn(conn) { // Precompile our statements conn.prepare("v7_station_select_fixed", "SELECT id FROM station WHERE rep=$1::int4 AND lat=$2::int4 AND lon=$3::int4 AND ident IS NULL"); conn.prepare("v7_station_select_mobile", "SELECT id FROM station WHERE rep=$1::int4 AND lat=$2::int4 AND lon=$3::int4 AND ident=$4::text"); conn.prepare("v7_station_insert", "INSERT INTO station (id, rep, lat, lon, ident) VALUES (DEFAULT, $1::int4, $2::int4, $3::int4, $4::text) RETURNING id"); conn.prepare("v7_station_select_station_data", "SELECT rep, lat, lon, ident FROM station WHERE id=$1::int4"); conn.prepare("v7_station_get_station_vars", R"( SELECT d.code, d.value, d.attrs FROM station_data d WHERE d.id_station=$1::int4 ORDER BY d.code )"); conn.prepare("v7_station_add_station_vars", R"( SELECT d.code, d.value FROM station_data d WHERE d.id_station = $1::int4 )"); } PostgreSQLStation::~PostgreSQLStation() { } DBStation PostgreSQLStation::lookup(Tracer<>& trc, int id_station) { using namespace dballe::sql::postgresql; Tracer<> trc_sel; Result res(conn.exec_prepared("v7_station_select_station_data", id_station)); if (trc) trc_sel.reset(trc->trace_select("v7_station_select_station_data", res.rowcount())); unsigned rows = res.rowcount(); if (trc_sel) trc_sel->add_row(rows); switch (rows) { case 0: { stringstream msg; msg << "Station with id " << id_station << " not found"; throw std::runtime_error(msg.str()); } case 1: { DBStation station; station.id = id_station; station.report = tr.repinfo().get_rep_memo(res.get_int4(0, 0)); station.coords.lat = res.get_int4(0, 1); station.coords.lon = res.get_int4(0, 2); if (res.is_null(0, 3)) station.ident.clear(); else station.ident = res.get_string(0, 3); return station; } default: error_consistency::throwf("select station data query returned %u results", rows); } } int PostgreSQLStation::maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) { using namespace dballe::sql::postgresql; int rep = tr.repinfo().obtain_id(st.report.c_str()); Tracer<> trc_sel; Result res; if (st.ident.get()) { if (trc) trc_sel.reset(trc->trace_select("v7_station_select_mobile", res.rowcount())); res = move(conn.exec_prepared("v7_station_select_mobile", rep, st.coords.lat, st.coords.lon, st.ident.get())); } else { if (trc) trc_sel.reset(trc->trace_select("v7_station_select_fixed")); res = move(conn.exec_prepared("v7_station_select_fixed", rep, st.coords.lat, st.coords.lon)); } unsigned rows = res.rowcount(); if (trc_sel) trc_sel->add_row(rows); switch (rows) { case 0: return MISSING_INT; case 1: return res.get_int4(0, 0); default: error_consistency::throwf("select station ID query returned %u results", rows); } } int PostgreSQLStation::insert_new(Tracer<>& trc, const dballe::DBStation& desc) { // If no station was found, insert a new one int rep = tr.repinfo().get_id(desc.report.c_str()); Tracer<> trc_ins(trc ? trc->trace_insert("v7_station_insert", 1) : nullptr); return conn.exec_prepared_one_row("v7_station_insert", rep, desc.coords.lat, desc.coords.lon, desc.ident.get()).get_int4(0, 0); } void PostgreSQLStation::get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) { using namespace dballe::sql::postgresql; TRACE("get_station_vars Performing query v7_station_get_station_vars with idst %d\n", id_station); Tracer<> trc_sel(trc ? trc->trace_select("v7_station_get_station_vars") : nullptr); Result res(conn.exec_prepared("v7_station_get_station_vars", id_station)); if (trc_sel) trc_sel->add_row(res.rowcount()); // Retrieve results for (unsigned row = 0; row < res.rowcount(); ++row) { Varcode code = res.get_int4(row, 0); TRACE("get_station_vars Got %01d%02d%03d %s\n", WR_VAR_FXY(code), res.get_string(row, 1)); unique_ptr var = newvar(code, res.get_string(row, 1)); if (!res.is_null(row, 2)) { TRACE("get_station_vars new attribute\n"); DBValues::decode(res.get_bytea(row, 2), [&](unique_ptr a) { var->seta(move(a)); }); } dest(move(var)); }; } void PostgreSQLStation::add_station_vars(Tracer<>& trc, int id_station, DBValues& values) { using namespace dballe::sql::postgresql; Tracer<> trc_sel(trc ? trc->trace_select("v7_station_add_station_vars") : nullptr); Result res(conn.exec_prepared("v7_station_add_station_vars", id_station)); if (trc_sel) trc_sel->add_row(res.rowcount()); for (unsigned row = 0; row < res.rowcount(); ++row) values.set(newvar((Varcode)res.get_int4(row, 0), res.get_string(row, 1))); } void PostgreSQLStation::run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function dest) { using namespace dballe::sql::postgresql; Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); // Start the query asynchronously int res; if (qb.bind_in_ident) { const char* args[1] = { qb.bind_in_ident }; res = PQsendQueryParams(conn, qb.sql_query.c_str(), 1, nullptr, args, nullptr, nullptr, 1); } else { res = PQsendQueryParams(conn, qb.sql_query.c_str(), 0, nullptr, nullptr, nullptr, nullptr, 1); } if (!res) throw sql::error_postgresql(conn, "executing " + qb.sql_query); dballe::DBStation station; conn.run_single_row_mode(qb.sql_query, [&](const Result& res) { if (trc_sel) trc_sel->add_row(res.rowcount()); for (unsigned row = 0; row < res.rowcount(); ++row) { station.id = res.get_int4(row, 0); station.report = tr.repinfo().get_rep_memo(res.get_int4(row, 1)); station.coords.lat = res.get_int4(row, 2); station.coords.lon = res.get_int4(row, 3); if (res.is_null(row, 4)) station.ident.clear(); else station.ident = res.get_string(row, 4); dest(station); } }); } void PostgreSQLStation::_dump(std::function out) { auto res = conn.exec("SELECT id, rep, lat, lon, ident FROM station"); for (unsigned row = 0; row < res.rowcount(); ++row) { const char* ident = res.is_null(row, 4) ? nullptr : res.get_string(row, 4); out(res.get_int4(row, 0), res.get_int4(row, 1), Coords((int)res.get_int4(row, 2), (int)res.get_int4(row, 3)), ident); } } } } } } dballe-8.6/dballe/db/v7/postgresql/driver.cc0000644000175000017500000001146113554564112015715 00000000000000#include "driver.h" #include "repinfo.h" #include "station.h" #include "levtr.h" #include "data.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/sql/postgresql.h" #include "dballe/var.h" #include #include using namespace std; using namespace wreport; using dballe::sql::PostgreSQLConnection; using dballe::sql::error_postgresql; namespace dballe { namespace db { namespace v7 { namespace postgresql { Driver::Driver(PostgreSQLConnection& conn) : v7::Driver(conn), conn(conn) { } Driver::~Driver() { } std::unique_ptr Driver::create_repinfo(v7::Transaction& tr) { return unique_ptr(new PostgreSQLRepinfo(conn)); } std::unique_ptr Driver::create_station(v7::Transaction& tr) { return unique_ptr(new PostgreSQLStation(tr, conn)); } std::unique_ptr Driver::create_levtr(v7::Transaction& tr) { return unique_ptr(new PostgreSQLLevTr(tr, conn)); } std::unique_ptr Driver::create_station_data(v7::Transaction& tr) { return unique_ptr(new PostgreSQLStationData(tr, conn)); } std::unique_ptr Driver::create_data(v7::Transaction& tr) { return unique_ptr(new PostgreSQLData(tr, conn)); } void Driver::create_tables_v7() { conn.exec_no_data(R"( CREATE TABLE repinfo ( id INTEGER PRIMARY KEY, memo VARCHAR(30) NOT NULL, description VARCHAR(255) NOT NULL, prio INTEGER NOT NULL, descriptor CHAR(6) NOT NULL, tablea INTEGER NOT NULL ); )"); conn.exec_no_data("CREATE UNIQUE INDEX ri_memo_uniq ON repinfo(memo);"); conn.exec_no_data("CREATE UNIQUE INDEX ri_prio_uniq ON repinfo(prio);"); conn.exec_no_data(R"( CREATE TABLE station ( id SERIAL PRIMARY KEY, rep INTEGER NOT NULL REFERENCES repinfo (id) ON DELETE CASCADE, lat INTEGER NOT NULL, lon INTEGER NOT NULL, ident VARCHAR(64) ); )"); conn.exec_no_data("CREATE UNIQUE INDEX pa_uniq ON station(rep, lat, lon, ident);"); conn.exec_no_data("CREATE INDEX pa_lon ON station(lon);"); conn.exec_no_data(R"( CREATE TABLE levtr ( id SERIAL PRIMARY KEY, ltype1 INTEGER NOT NULL, l1 INTEGER NOT NULL, ltype2 INTEGER NOT NULL, l2 INTEGER NOT NULL, pind INTEGER NOT NULL, p1 INTEGER NOT NULL, p2 INTEGER NOT NULL ); )"); conn.exec_no_data("CREATE UNIQUE INDEX levtr_uniq ON levtr(ltype1, l1, ltype2, l2, pind, p1, p2);"); conn.exec_no_data(R"( CREATE TABLE station_data ( id SERIAL PRIMARY KEY, id_station INTEGER NOT NULL REFERENCES station (id) ON DELETE CASCADE, code INTEGER NOT NULL, value VARCHAR(255) NOT NULL, attrs BYTEA ); )"); conn.exec_no_data("CREATE UNIQUE INDEX station_data_uniq on station_data(id_station, code);"); conn.exec_no_data(R"( CREATE TABLE data ( id SERIAL PRIMARY KEY, id_station INTEGER NOT NULL REFERENCES station (id) ON DELETE CASCADE, id_levtr INTEGER NOT NULL REFERENCES levtr(id) ON DELETE CASCADE, datetime TIMESTAMP NOT NULL, code INTEGER NOT NULL, value VARCHAR(255) NOT NULL, attrs BYTEA ); )"); conn.exec_no_data("CREATE UNIQUE INDEX data_uniq on data(id_station, datetime, id_levtr, code);"); // When possible, replace with a postgresql 9.5 BRIN index conn.exec_no_data("CREATE INDEX data_dt ON data(datetime);"); conn.set_setting("version", "V7"); } void Driver::delete_tables_v7() { conn.drop_table_if_exists("data"); conn.drop_table_if_exists("station_data"); conn.drop_table_if_exists("levtr"); conn.drop_table_if_exists("station"); conn.drop_table_if_exists("repinfo"); conn.drop_settings(); } void Driver::vacuum_v7() { conn.exec_no_data(R"( DELETE FROM levtr WHERE id IN ( SELECT ltr.id FROM levtr ltr LEFT JOIN data d ON d.id_levtr = ltr.id WHERE d.id_levtr is NULL) )"); conn.exec_no_data(R"( DELETE FROM station_data WHERE id IN ( SELECT sd.id FROM station_data sd LEFT JOIN data dd ON sd.id_station = dd.id_station WHERE dd.id IS NULL) )"); conn.exec_no_data(R"( DELETE FROM station WHERE id IN ( SELECT p.id FROM station p LEFT JOIN data d ON d.id_station = p.id WHERE d.id is NULL) )"); } } } } } dballe-8.6/dballe/db/v7/utils.cc0000644000175000017500000000011513554564112013351 00000000000000#include "utils.h" namespace dballe { namespace db { namespace v7 { } } } dballe-8.6/dballe/db/v7/cache-test.cc0000644000175000017500000000200213554564112014226 00000000000000#include "dballe/core/tests.h" #include "cache.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("db_v7_cache"); void Tests::register_tests() { add_method("levtr", [] { db::v7::LevTrCache cache; wassert_false(cache.find_entry(1)); wassert_false(cache.find_entry(MISSING_INT)); wassert(actual(cache.find_id(db::v7::LevTrEntry())) == MISSING_INT); db::v7::LevTrEntry lt; lt.id = 1; lt.level = Level(1); lt.trange = Trange(4, 2, 2); cache.insert(lt); wassert_true(cache.find_entry(1)); wassert(actual(*cache.find_entry(1)) == lt); wassert(actual(cache.find_id(lt)) == 1); cache.insert(lt); wassert_true(cache.find_entry(1)); wassert(actual(*cache.find_entry(1)) == lt); wassert(actual(cache.find_id(lt)) == 1); wassert(actual(cache.reverse[lt.level].size()) == 1u); }); } } dballe-8.6/dballe/db/v7/batch.h0000644000175000017500000001105613554564112013142 00000000000000#ifndef DBALLE_DB_V7_BATCH_H #define DBALLE_DB_V7_BATCH_H #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { struct Transaction; class Batch { protected: bool write_attrs = true; batch::Station* last_station = nullptr; bool have_station(const std::string& report, const Coords& coords, const Ident& ident); void new_station(Tracer<>& trc, const std::string& report, const Coords& coords, const Ident& ident); public: Transaction& transaction; unsigned count_select_stations = 0; unsigned count_select_station_data = 0; unsigned count_select_data = 0; Batch(Transaction& transaction) : transaction(transaction) {} ~Batch(); void set_write_attrs(bool write_attrs); batch::Station* get_station(Tracer<>& trc, const dballe::DBStation& station, bool station_can_add); batch::Station* get_station(Tracer<>& trc, const std::string& report, const Coords& coords, const Ident& ident); void write_pending(Tracer<>& trc); void clear(); void dump(FILE* out) const; }; namespace batch { enum UpdateMode { UPDATE, IGNORE, ERROR, }; struct StationDatum { int id = MISSING_INT; const wreport::Var* var; StationDatum(const wreport::Var* var) : var(var) {} StationDatum(int id, const wreport::Var* var) : id(id), var(var) {} void dump(FILE* out) const; bool operator<(const StationDatum& o) const { return var->code() < o.var->code(); } bool operator==(const StationDatum& o) const { return var->code() == o.var->code(); } }; inline const wreport::Varcode& station_data_ids_get_value(const IdVarcode& item) { return item.varcode; } struct StationDataIDs : public core::SmallSet { }; struct StationData { StationDataIDs ids_by_code; std::vector to_insert; std::vector to_update; bool loaded = false; void add(const wreport::Var* var, UpdateMode on_conflict); void write_pending(Tracer<>& trc, Transaction& tr, int station_id, bool with_attrs); }; struct MeasuredDatum { int id = MISSING_INT; int id_levtr; const wreport::Var* var; MeasuredDatum(int id_levtr, const wreport::Var* var) : id_levtr(id_levtr), var(var) {} MeasuredDatum(int id, int id_levtr, const wreport::Var* var) : id(id), id_levtr(id_levtr), var(var) {} void dump(FILE* out) const; bool operator<(const MeasuredDatum& o) const { return id_levtr < o.id_levtr || (id_levtr == o.id_levtr && var->code() < o.var->code()); } bool operator==(const MeasuredDatum& o) const { return id_levtr == o.id_levtr && var->code() == o.var->code(); } }; struct MeasuredDataID { IdVarcode id_varcode; int id; MeasuredDataID(IdVarcode id_varcode, int id) : id_varcode(id_varcode), id(id) { } }; inline const IdVarcode& measured_data_ids_get_value(const MeasuredDataID& item) { return item.id_varcode; } struct MeasuredDataIDs : public core::SmallSet { }; struct MeasuredData { Datetime datetime; MeasuredDataIDs ids_on_db; std::vector to_insert; std::vector to_update; MeasuredData(Datetime datetime) : datetime(datetime) { } void add(int id_levtr, const wreport::Var* var, UpdateMode on_conflict); void write_pending(Tracer<>& trc, Transaction& tr, int station_id, bool with_attrs); }; inline const Datetime& measured_data_vector_get_value(MeasuredData* const& item) { return item->datetime; } struct MeasuredDataVector : public core::SmallSet { MeasuredDataVector() {} MeasuredDataVector(const MeasuredDataVector&) = delete; MeasuredDataVector(MeasuredDataVector&&) = default; ~MeasuredDataVector(); MeasuredDataVector& operator=(const MeasuredDataVector&) = delete; MeasuredDataVector& operator=(MeasuredDataVector&&) = default; }; struct Station : public dballe::DBStation { Batch& batch; bool is_new = true; StationData station_data; MeasuredDataVector measured_data; Station(Batch& batch) : batch(batch) {} StationData& get_station_data(Tracer<>& trc); MeasuredData& get_measured_data(Tracer<>& trc, const Datetime& datetime); void write_pending(Tracer<>& trc, bool with_attrs); void dump(FILE* out) const; }; } } } } #endif dballe-8.6/dballe/db/v7/cursor.cc0000644000175000017500000002535013554564112013536 00000000000000#include "cursor.h" #include "qbuilder.h" #include "db.h" #include "transaction.h" #include "dballe/sql/sql.h" #include #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/station.h" #include "dballe/db/v7/levtr.h" #include "dballe/db/v7/data.h" #include "dballe/types.h" #include "dballe/var.h" #include "dballe/core/var.h" #include "dballe/core/data.h" #include "dballe/core/query.h" #include "wreport/var.h" #include #include #include namespace { // Consts used for to_record_todo const unsigned int TOREC_PSEUDOANA = 1 << 0; const unsigned int TOREC_BASECONTEXT = 1 << 1; const unsigned int TOREC_DATACONTEXT = 1 << 2; const unsigned int TOREC_DATA = 1 << 3; } using namespace std; using namespace wreport; namespace dballe { namespace db { namespace v7 { namespace cursor { void StationRows::load(Tracer<>& trc, const StationQueryBuilder& qb) { results.clear(); tr->station().run_station_query(trc, qb, [&](const dballe::DBStation& desc) { results.emplace_back(desc); }); at_start = true; cur = results.begin(); } const DBValues& StationRows::values() const { if (!cur->values.get()) { cur->values.reset(new DBValues); Tracer<> trc(tr->trc ? tr->trc->trace_add_station_vars() : nullptr); // FIXME: this could be made more efficient by querying all matching // station values, and merging rows during load, so it would only do // one query to the database tr->station().add_station_vars(trc, cur->station.id, *cur->values); } return *cur->values; } void StationDataRows::load(Tracer<>& trc, const DataQueryBuilder& qb) { results.clear(); tr->station_data().run_station_data_query(trc, qb, [&](const dballe::DBStation& station, int id_data, std::unique_ptr var) { results.emplace_back(station, id_data, std::move(var)); }); at_start = true; cur = results.begin(); } void DataRows::load(Tracer<>& trc, const DataQueryBuilder& qb) { results.clear(); std::set ids; tr->data().run_data_query(trc, qb, [&](const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var) { results.emplace_back(station, id_levtr, datetime, id_data, std::move(var)); ids.insert(id_levtr); }); at_start = true; cur = results.begin(); tr->levtr().prefetch_ids(trc, ids); } bool DataRows::add_to_best_results(const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var) { int prio = tr->repinfo().get_priority(station.report); if (results.empty()) goto append; if (station.coords != results.back().station.coords) goto append; if (station.ident != results.back().station.ident) goto append; if (id_levtr != results.back().id_levtr) goto append; if (datetime != results.back().datetime) goto append; if (var->code() != results.back().value.code()) goto append; if (prio <= insert_cur_prio) return false; // Replace results.back().station = station; results.back().value = DBValue(id_data, std::move(var)); insert_cur_prio = prio; return true; append: results.emplace_back(station, id_levtr, datetime, id_data, std::move(var)); insert_cur_prio = prio; return true; } void DataRows::load_best(Tracer<>& trc, const DataQueryBuilder& qb) { results.clear(); set ids; tr->data().run_data_query(trc, qb, [&](const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var) { if (add_to_best_results(station, id_levtr, datetime, id_data, move(var))) ids.insert(id_levtr); }); at_start = true; cur = results.begin(); tr->levtr().prefetch_ids(trc, ids); } void SummaryRows::load(Tracer<>& trc, const SummaryQueryBuilder& qb) { results.clear(); set ids; tr->data().run_summary_query(trc, qb, [&](const dballe::DBStation& station, int id_levtr, wreport::Varcode code, const DatetimeRange& datetime, size_t count) { results.emplace_back(station, id_levtr, code, datetime, count); ids.insert(id_levtr); }); at_start = true; cur = results.begin(); tr->levtr().prefetch_ids(trc, ids); } template int Base::remaining() const { if (rows.at_start) return rows.results.size(); else return rows.results.end() - rows.cur - 1; } template void Base::discard() { rows.discard(); } template unsigned Base::test_iterate(FILE* dump) { unsigned count; for (count = 0; next(); ++count) if (dump) rows->dump(dump); return count; } template class Base; template class Base; template class Base; template class Base; void StationRow::dump(FILE* out) const { fprintf(out, "%02d %8.8s %02.4f %02.4f %-10s\n", station.id, station.report.c_str(), station.coords.dlat(), station.coords.dlon(), station.ident.get()); } void StationDataRow::dump(FILE* out) const { fprintf(out, "%02d %8.8s %02.4f %02.4f %-10s ", station.id, station.report.c_str(), station.coords.dlat(), station.coords.dlon(), station.ident.get()); value.print(out); } void DataRow::dump(FILE* out) const { fprintf(out, "%02d %8.8s %02.4f %02.4f %-10s %4d ", station.id, station.report.c_str(), station.coords.dlat(), station.coords.dlon(), station.ident.get(), id_levtr); datetime.print_iso8601(out, ' '); fprintf(out, " "); value.print(out); } void SummaryRow::dump(FILE* out) const { fprintf(out, "%02d %8.8s %02.4f %02.4f %-10s %4d %d%02d%03d\n", station.id, station.report.c_str(), station.coords.dlat(), station.coords.dlon(), station.ident.get(), id_levtr, WR_VAR_FXY(code)); } DBValues Stations::get_values() const { return rows.values(); } void Stations::remove() { core::Query query; query.ana_id = rows->station.id; rows.tr->remove_station_data(query); rows.tr->remove_data(query); } StationData::StationData(DataQueryBuilder& qb, bool with_attributes) : Base(qb.tr), with_attributes(with_attributes) {} void StationData::query_attrs(std::function)> dest, bool force_read) { if (!force_read && with_attributes) { for (const wreport::Var* a = rows->value->next_attr(); a != NULL; a = a->next_attr()) dest(std::unique_ptr(new Var(*a))); } else { rows.tr->attr_query_station(attr_reference_id(), dest); } } void StationData::remove() { rows.tr->remove_station_data_by_id(rows->value.data_id); } Data::Data(DataQueryBuilder& qb, bool with_attributes) : Base(qb.tr), with_attributes(with_attributes) {} void Data::query_attrs(std::function)> dest, bool force_read) { if (!force_read && with_attributes) { for (const Var* a = rows->value->next_attr(); a != NULL; a = a->next_attr()) dest(std::unique_ptr(new Var(*a))); } else { rows.tr->attr_query_data(attr_reference_id(), dest); } } void Data::remove() { rows.tr->remove_data_by_id(rows->value.data_id); } void Summary::remove() { core::Query query; query.ana_id = rows->station.id; auto levtr = rows.get_levtr(); query.level = levtr.level; query.trange = levtr.trange; query.varcodes.insert(rows->code); rows.tr->remove_data(query); } unique_ptr run_station_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& q, bool explain) { unsigned int modifiers = q.get_modifiers(); StationQueryBuilder qb(tr, q, modifiers); qb.build(); if (explain) { fprintf(stderr, "EXPLAIN "); q.print(stderr); tr->db->conn->explain(qb.sql_query, stderr); } auto resptr = new Stations(tr); unique_ptr res(resptr); resptr->rows.load(trc, qb); return std::move(res); } unique_ptr run_station_data_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& q, bool explain) { unsigned int modifiers = q.get_modifiers(); DataQueryBuilder qb(tr, q, modifiers, true); qb.build(); if (explain) { fprintf(stderr, "EXPLAIN "); q.print(stderr); tr->db->conn->explain(qb.sql_query, stderr); } unique_ptr res; if (modifiers & DBA_DB_MODIFIER_BEST) { throw error_unimplemented("best queries of station vars"); //auto resptr = new Best(tr, modifiers); //res.reset(resptr); //resptr->load(qb); } else { auto resptr = new StationData(qb, modifiers & DBA_DB_MODIFIER_WITH_ATTRIBUTES); res.reset(resptr); resptr->rows.load(trc, qb); } return std::move(res); } unique_ptr run_data_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& q, bool explain) { unsigned int modifiers = q.get_modifiers(); DataQueryBuilder qb(tr, q, modifiers, false); qb.build(); if (explain) { fprintf(stderr, "EXPLAIN "); q.print(stderr); tr->db->conn->explain(qb.sql_query, stderr); } unique_ptr res; auto resptr = new Data(qb, modifiers & DBA_DB_MODIFIER_WITH_ATTRIBUTES); res.reset(resptr); if (modifiers & DBA_DB_MODIFIER_BEST) resptr->rows.load_best(trc, qb); else resptr->rows.load(trc, qb); return std::move(res); } unique_ptr run_summary_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& q, bool explain) { unsigned int modifiers = q.get_modifiers(); if (modifiers & DBA_DB_MODIFIER_BEST) throw error_consistency("cannot use query=best on summary queries"); SummaryQueryBuilder qb(tr, q, modifiers, false); qb.build(); if (explain) { fprintf(stderr, "EXPLAIN "); q.print(stderr); tr->db->conn->explain(qb.sql_query, stderr); } auto resptr = new Summary(tr); unique_ptr res(resptr); resptr->rows.load(trc, qb); return std::move(res); } void run_delete_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& q, bool station_vars, bool explain) { unsigned int modifiers = q.get_modifiers(); if (modifiers & DBA_DB_MODIFIER_BEST) throw error_consistency("cannot use query=best on delete queries"); IdQueryBuilder qb(tr, q, modifiers, station_vars); qb.build(); if (explain) { fprintf(stderr, "EXPLAIN "); q.print(stderr); tr->db->conn->explain(qb.sql_query, stderr); } if (station_vars) tr->station_data().remove(trc, qb); else tr->data().remove(trc, qb); } } } } } dballe-8.6/dballe/db/v7/cursor-access.cc0000644000175000017500000010701713554574017015003 00000000000000#include "cursor.h" #include "repinfo.h" #include "levtr.h" #include using namespace wreport; namespace dballe { namespace db { namespace v7 { namespace cursor { /* * Stations */ void StationRows::enq(impl::Enq& enq) const { if (enq.search_b_values(values())) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 3: if (key[0] == 'l') { switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(cur->station.coords.lat); } else { enq.search_alias_values(values()); } break; case 'o': if (key[2] == 'n') { enq.set_lon(cur->station.coords.lon); } else { enq.search_alias_values(values()); } break; default: enq.search_alias_values(values()); } } else { enq.search_alias_values(values()); } break; case 5: if (memcmp(key + 0, "ident", 5) == 0) { enq.set_ident(cur->station.ident); } else { enq.search_alias_values(values()); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_values(values()); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(cur->station.id); } else { enq.search_alias_values(values()); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!cur->station.ident.is_missing()); } else { enq.search_alias_values(values()); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(cur->station.coords); } else { enq.search_alias_values(values()); } break; default: enq.search_alias_values(values()); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(cur->station); } else { enq.search_alias_values(values()); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { enq.set_int(get_priority()); } else { enq.search_alias_values(values()); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_values(values()); } break; default: enq.search_alias_values(values()); } break; default: enq.search_alias_values(values()); } } /* * StationData */ void StationDataRows::enq(impl::Enq& enq) const { if (enq.search_b_value(cur->value)) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(cur->station.coords.lat); } else { enq.search_alias_value(cur->value); } break; case 'o': if (key[2] == 'n') { enq.set_lon(cur->station.coords.lon); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->value.code()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(cur->station.ident); } else { enq.search_alias_value(cur->value); } break; case 'a': if (memcmp(key + 1, "ttrs", 4) == 0) { enq.set_attrs(cur->value.get()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_value(cur->value); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(cur->station.id); } else { enq.search_alias_value(cur->value); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!cur->station.ident.is_missing()); } else { enq.search_alias_value(cur->value); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(cur->station.coords); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(cur->station); } else { enq.search_alias_value(cur->value); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { enq.set_int(get_priority()); } else { enq.search_alias_value(cur->value); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_value(cur->value); } break; case 'v': if (memcmp(key + 1, "ariable", 7) == 0) { enq.set_var(cur->value.get()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 10: if (memcmp(key + 0, "context_id", 10) == 0) { enq.set_dballe_int(cur->value.data_id); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } } /* * Data */ void BaseDataRows::enq(impl::Enq& enq) const { if (enq.search_b_value(cur->value)) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': enq.set_dballe_int(get_levtr().level.l1); break; case '2': enq.set_dballe_int(get_levtr().level.l2); break; default: enq.search_alias_value(cur->value); } break; case 'p': switch (key[1]) { case '1': enq.set_dballe_int(get_levtr().trange.p1); break; case '2': enq.set_dballe_int(get_levtr().trange.p2); break; default: enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(cur->station.coords.lat); } else { enq.search_alias_value(cur->value); } break; case 'o': if (key[2] == 'n') { enq.set_lon(cur->station.coords.lon); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 'd': if (memcmp(key + 1, "ay", 2) == 0) { enq.set_int(cur->datetime.day); } else { enq.search_alias_value(cur->value); } break; case 'm': if (memcmp(key + 1, "in", 2) == 0) { enq.set_int(cur->datetime.minute); } else { enq.search_alias_value(cur->value); } break; case 's': if (memcmp(key + 1, "ec", 2) == 0) { enq.set_int(cur->datetime.second); } else { enq.search_alias_value(cur->value); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->value.code()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { enq.set_int(cur->datetime.year); } else { enq.search_alias_value(cur->value); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { enq.set_int(cur->datetime.hour); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(cur->station.ident); } else { enq.search_alias_value(cur->value); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { enq.set_int(cur->datetime.month); } else { enq.search_alias_value(cur->value); } break; case 'l': if (memcmp(key + 1, "evel", 4) == 0) { enq.set_level(get_levtr().level); } else { enq.search_alias_value(cur->value); } break; case 'a': if (memcmp(key + 1, "ttrs", 4) == 0) { enq.set_attrs(cur->value.get()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_value(cur->value); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(cur->station.id); } else { enq.search_alias_value(cur->value); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!cur->station.ident.is_missing()); } else { enq.search_alias_value(cur->value); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(cur->station.coords); } else { enq.search_alias_value(cur->value); } break; case 't': if (memcmp(key + 1, "range", 5) == 0) { enq.set_trange(get_levtr().trange); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(cur->station); } else { enq.search_alias_value(cur->value); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { enq.set_int(get_priority()); } else { enq.search_alias_value(cur->value); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(cur->station.report); } else { enq.search_alias_value(cur->value); } break; case 'd': if (memcmp(key + 1, "atetime", 7) == 0) { enq.set_datetime(cur->datetime); } else { enq.search_alias_value(cur->value); } break; case 'v': if (memcmp(key + 1, "ariable", 7) == 0) { enq.set_var(cur->value.get()); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': enq.set_dballe_int(get_levtr().level.ltype1); break; case '2': enq.set_dballe_int(get_levtr().level.ltype2); break; default: enq.search_alias_value(cur->value); } } else { enq.search_alias_value(cur->value); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { enq.set_dballe_int(get_levtr().trange.pind); } else { enq.search_alias_value(cur->value); } break; case 'c': if (memcmp(key + 1, "ontext_id", 9) == 0) { enq.set_dballe_int(cur->value.data_id); } else { enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } break; default: enq.search_alias_value(cur->value); } } /* * Summary */ void SummaryRows::enq(impl::Enq& enq) const { const auto key = enq.key; const auto len = enq.len; switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': enq.set_dballe_int(get_levtr().level.l1); break; case '2': enq.set_dballe_int(get_levtr().level.l2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'p': switch (key[1]) { case '1': enq.set_dballe_int(get_levtr().trange.p1); break; case '2': enq.set_dballe_int(get_levtr().trange.p2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(cur->station.coords.lat); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'o': if (key[2] == 'n') { enq.set_lon(cur->station.coords.lon); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->code); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(cur->station.ident); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'l': if (memcmp(key + 1, "evel", 4) == 0) { enq.set_level(get_levtr().level); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "ount", 4) == 0) { enq.set_int(cur->count); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(cur->station.report); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(cur->station.id); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!cur->station.ident.is_missing()); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(cur->station.coords); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'd': if (memcmp(key + 1, "aym", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.day); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[5] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.day); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 's': if (memcmp(key + 1, "ecm", 3) == 0) { switch (key[4]) { case 'a': if (key[5] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.second); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[5] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.second); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 't': if (memcmp(key + 1, "range", 5) == 0) { enq.set_trange(get_levtr().trange); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 7: switch (key[0]) { case 's': if (memcmp(key + 1, "tation", 6) == 0) { enq.set_station(cur->station); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'y': if (memcmp(key + 1, "earm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.year); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'h': if (memcmp(key + 1, "ourm", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.hour); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.hour); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "inum", 4) == 0) { switch (key[5]) { case 'a': if (key[6] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.minute); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[6] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.minute); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { enq.set_int(get_priority()); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(cur->station.report); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'm': if (memcmp(key + 1, "onthm", 5) == 0) { switch (key[6]) { case 'a': if (key[7] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.month); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[7] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.month); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': enq.set_dballe_int(get_levtr().level.ltype1); break; case '2': enq.set_dballe_int(get_levtr().level.ltype2); break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { enq.set_dballe_int(get_levtr().trange.pind); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'c': if (memcmp(key + 1, "ontext_id", 9) == 0) { enq.set_int(cur->count); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 11: if (memcmp(key + 0, "datetimem", 9) == 0) { switch (key[9]) { case 'a': if (key[10] == 'x') { if (cur->dtrange.is_missing()) return; else enq.set_datetime(cur->dtrange.max); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; case 'i': if (key[10] == 'n') { if (cur->dtrange.is_missing()) return; else enq.set_datetime(cur->dtrange.min); } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } else { wreport::error_notfound::throwf("key %s not found on this query result", key); } break; default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } } } } } dballe-8.6/dballe/db/v7/cache.cc0000644000175000017500000000536013554564112013263 00000000000000#include "cache.h" #include using namespace std; namespace dballe { namespace db { namespace v7 { bool LevTrEntry::operator==(const LevTrEntry& o) const { if (id != MISSING_INT && o.id != MISSING_INT) return id == o.id; if (level != o.level) return false; return trange == o.trange; } bool LevTrEntry::operator!=(const LevTrEntry& o) const { if (id != MISSING_INT && o.id != MISSING_INT) return id != o.id; if (level == o.level) return false; return trange != o.trange; } std::ostream& operator<<(std::ostream& out, const LevTrEntry& l) { out << "("; if (l.id == MISSING_INT) out << "-"; else out << l.id; return out << ":" << l.level << ":" << l.trange; } int LevTrReverseIndex::find_id(const LevTrEntry& lt) const { auto li = find(lt.level); if (li == end()) return MISSING_INT; for (auto i: li->second) if (i->level == lt.level && i->trange == lt.trange) return i->id; return MISSING_INT; } void LevTrReverseIndex::add(const LevTrEntry* lt) { auto li = find(lt->level); if (li == end()) insert(make_pair(lt->level, std::vector{lt})); else li->second.push_back(lt); } LevTrCache::~LevTrCache() { for (auto& i: by_id) delete i.second; } void LevTrCache::clear() { for (auto& i: by_id) delete i.second; by_id.clear(); reverse.clear(); } const LevTrEntry* LevTrCache::find_entry(int id) const { auto i = by_id.find(id); if (i == by_id.end()) return nullptr; return i->second; } const LevTrEntry* LevTrCache::insert(const LevTrEntry& e) { return insert(std::unique_ptr(new LevTrEntry(e))); } const LevTrEntry* LevTrCache::insert(const LevTrEntry& e, int id) { std::unique_ptr ne(new LevTrEntry(e)); ne->id = id; return insert(move(ne)); } const LevTrEntry* LevTrCache::insert(std::unique_ptr e) { if (e->id == MISSING_INT) throw std::runtime_error("station to cache in transaction state must have a database ID"); auto i = this->by_id.find(e->id); if (i != this->by_id.end()) { // StationCache do not move: if we have a match on the ID, we just need to // enforce that there is no mismatch on the station data if (*i->second != *e) throw std::runtime_error("cannot replace a cached DB entry with one with the same ID and different data"); return i->second; } const LevTrEntry* res = e.get(); this->by_id.insert(make_pair(res->id, e.release())); reverse.add(res); return res; } int LevTrCache::find_id(const LevTrEntry& e) const { if (e.id != MISSING_INT) return e.id; return reverse.find_id(e); } } } } dballe-8.6/dballe/db/v7/db.cc0000644000175000017500000000506613554564112012610 00000000000000#include "db.h" #include "dballe/sql/sql.h" #include "dballe/sql/querybuf.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/station.h" #include "dballe/db/v7/levtr.h" #include "dballe/db/v7/data.h" #include "cursor.h" #include "dballe/core/query.h" #include "dballe/types.h" #include #include #include #include #include using namespace std; using namespace wreport; using dballe::sql::Connection; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { // First part of initialising a dba_db DB::DB(unique_ptr conn) : conn(conn.release()), m_driver(v7::Driver::create(*this->conn).release()) { if (getenv("DBA_EXPLAIN") != NULL) explain_queries = true; if (const char* logdir = getenv("DBA_PROFILE")) trace = new CollectTrace(logdir); else if (Trace::in_test_suite()) trace = new QuietCollectTrace; else trace = new NullTrace; auto trc = trace->trace_connect(this->conn->get_url()); /* Set the connection timeout */ /* SQLSetConnectAttr(pc.od_conn, SQL_LOGIN_TIMEOUT, (SQLPOINTER *)5, 0); */ } DB::~DB() { trace->save(); delete m_driver; delete conn; delete trace; } v7::Driver& DB::driver() { return *m_driver; } std::shared_ptr DB::transaction(bool readonly) { auto res = conn->transaction(readonly); return make_shared(dynamic_pointer_cast(shared_from_this()), move(res)); } std::shared_ptr DB::test_transaction(bool readonly) { auto res = conn->transaction(readonly); return make_shared(dynamic_pointer_cast(shared_from_this()), move(res)); } void DB::delete_tables() { m_driver->delete_tables_v7(); } void DB::disappear() { // TODO: track open trasnsactions with weak pointers and roll them all // back, or raise errors if some of them have not been fired yet? m_driver->delete_tables_v7(); } void DB::reset(const char* repinfo_file) { auto trc = trace->trace_reset(repinfo_file); disappear(); m_driver->create_tables_v7(); // Populate the tables with values auto tr = dynamic_pointer_cast(transaction()); int added, deleted, updated; tr->update_repinfo(repinfo_file, &added, &deleted, &updated); tr->commit(); } void DB::vacuum() { auto trc = trace->trace_vacuum(); auto t = conn->transaction(); driver().vacuum_v7(); t->commit(); } } } } dballe-8.6/dballe/db/v7/levtr.h0000644000175000017500000000333113554564112013212 00000000000000#ifndef DBALLE_DB_V7_LEVTR_H #define DBALLE_DB_V7_LEVTR_H #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { /** * Precompiled queries to manipulate the lev_tr table */ struct LevTr { protected: v7::Transaction& tr; LevTrCache cache; virtual void _dump(std::function out) = 0; public: LevTr(v7::Transaction& tr); virtual ~LevTr(); /** * Invalidate the LevTrEntry cache. * * Further accesses will be done via the database, and slowly repopulate * the cache from scratch. */ void clear_cache(); /** * Given a set of IDs, load LevTr information for them and add it to the cache. */ virtual void prefetch_ids(Tracer<>& trc, const std::set& ids) = 0; /** * Get/create a Context in the Msg for this level/timerange. * * @returns the context, or 0 if the id is not valid. */ impl::msg::Context* to_msg(Tracer<>& trc, int id, impl::Message& msg); /** * Lookup a LevTr entry from the cache, throwing an exception if it is not found */ const LevTrEntry& lookup_cache(int id); /// Look up a LevTr from the database given its ID. virtual const LevTrEntry* lookup_id(Tracer<>& trc, int id) = 0; /** * Look up a LevTr from the database given its description. Insert a new * one if not found. */ virtual int obtain_id(Tracer<>& trc, const LevTrEntry& desc) = 0; /// Dump the entire contents of the table to an output stream void dump(FILE* out); }; } } } #endif dballe-8.6/dballe/db/v7/cache.h0000644000175000017500000000335013554564112013122 00000000000000#ifndef DBALLE_DB_V7_CACHE_H #define DBALLE_DB_V7_CACHE_H #include #include #include #include #include namespace dballe { namespace db { namespace v7 { struct LevTrEntry { // Database ID int id = MISSING_INT; /// Vertical level or layer Level level; /// Time range Trange trange; LevTrEntry() = default; LevTrEntry(int id, const Level& level, const Trange& trange) : id(id), level(level), trange(trange) {} LevTrEntry(const Level& level, const Trange& trange) : level(level), trange(trange) {} LevTrEntry(const LevTrEntry&) = default; LevTrEntry(LevTrEntry&&) = default; LevTrEntry& operator=(const LevTrEntry&) = default; LevTrEntry& operator=(LevTrEntry&&) = default; bool operator==(const LevTrEntry& o) const; bool operator!=(const LevTrEntry& o) const; }; std::ostream& operator<<(std::ostream&, const LevTrEntry&); struct LevTrReverseIndex : public std::unordered_map> { int find_id(const LevTrEntry& st) const; void add(const LevTrEntry* st); }; struct LevTrCache { std::unordered_map by_id; LevTrReverseIndex reverse; LevTrCache() = default; LevTrCache(const LevTrCache&) = delete; LevTrCache(LevTrCache&&) = delete; LevTrCache& operator=(const LevTrCache&) = delete; LevTrCache& operator=(LevTrCache&&) = delete; ~LevTrCache(); const LevTrEntry* find_entry(int id) const; const LevTrEntry* insert(const LevTrEntry& e); const LevTrEntry* insert(const LevTrEntry& e, int id); const LevTrEntry* insert(std::unique_ptr e); int find_id(const LevTrEntry& e) const; void clear(); }; } } } #endif dballe-8.6/dballe/db/v7/station.h0000644000175000017500000000374513554564112013550 00000000000000#ifndef DBALLE_DB_V7_STATION_H #define DBALLE_DB_V7_STATION_H #include #include #include #include #include #include #include #include #include #include #include namespace wreport { struct Var; } namespace dballe { namespace db { namespace v7 { struct Station { protected: v7::Transaction& tr; virtual void _dump(std::function out) = 0; public: Station(v7::Transaction& tr); virtual ~Station(); /// Lookup station data by ID virtual DBStation lookup(Tracer<>& trc, int id_station) = 0; /** * Get the station ID given latitude, longitude and mobile identifier. * * It returns MISSING_INT if it does not exist. */ virtual int maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) = 0; /** * Insert a new station in the database, without checking if it already exists. * * Returns the ID of the new station */ virtual int insert_new(Tracer<>& trc, const dballe::DBStation& desc) = 0; /** * Run a station query, iterating on the resulting stations */ virtual void run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function) = 0; /** * Export station variables */ virtual void get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) = 0; /** * Add all station variables (without attributes) to values. * * If the same variable exists in many different networks, the one with the * highest priority will be used. */ virtual void add_station_vars(Tracer<>& trc, int id_station, DBValues& values) = 0; /** * Dump the entire contents of the table to an output stream */ void dump(FILE* out); }; } } } #endif dballe-8.6/dballe/db/v7/cursor.h0000644000175000017500000002313413572414716013402 00000000000000#ifndef DBA_DB_V7_CURSOR_H #define DBA_DB_V7_CURSOR_H #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace cursor { struct Stations; struct StationData; struct Data; struct Summary; /** * Row resulting from a station query */ struct StationRow { dballe::DBStation station; mutable std::unique_ptr values; StationRow(const dballe::DBStation& station) : station(station) {} void dump(FILE* out) const; }; struct StationDataRow { dballe::DBStation station; DBValue value; StationDataRow(const dballe::DBStation& station, int id_data, std::unique_ptr var) : station(station), value(id_data, std::move(var)) {} StationDataRow(const StationDataRow&) = delete; StationDataRow(StationDataRow&& o) = default; StationDataRow& operator=(const StationDataRow&) = delete; StationDataRow& operator=(StationDataRow&& o) = default; ~StationDataRow() {} void dump(FILE* out) const; }; struct DataRow : public StationDataRow { int id_levtr; Datetime datetime; using StationDataRow::StationDataRow; DataRow(const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var) : StationDataRow(station, id_data, std::move(var)), id_levtr(id_levtr), datetime(datetime) {} void dump(FILE* out) const; }; struct SummaryRow { dballe::DBStation station; int id_levtr; wreport::Varcode code; DatetimeRange dtrange; size_t count = 0; SummaryRow(const dballe::DBStation& station, int id_levtr, wreport::Varcode code, const DatetimeRange& dtrange, size_t count) : station(station), id_levtr(id_levtr), code(code), dtrange(dtrange), count(count) {} void dump(FILE* out) const; }; template struct Rows { /// Database to operate on std::shared_ptr tr; /// Storage for the raw database results std::vector results; /// Iterator to the current position in results typename std::vector::const_iterator cur; /// True if we are at the start of the iteration bool at_start = true; Rows(std::shared_ptr tr) : tr(tr) {} const Row* operator->() const { return &*cur; } int get_priority() const { return tr->repinfo().get_priority(cur->station.report); } bool next() { if (at_start) at_start = false; else if (cur != results.end()) ++cur; return cur != results.end(); } void discard() { at_start = false; cur = results.end(); } }; struct StationRows : public Rows { using Rows::Rows; const DBValues& values() const; void load(Tracer<>& trc, const StationQueryBuilder& qb); void enq(impl::Enq& enq) const; }; struct StationDataRows : public Rows { using Rows::Rows; void load(Tracer<>& trc, const DataQueryBuilder& qb); void enq(impl::Enq& enq) const; }; template struct LevTrRows : public Rows { using Rows::Rows; // Cached levtr for the current row mutable const LevTrEntry* levtr = nullptr; bool next() { levtr = nullptr; return Rows::next(); } void discard() { levtr = nullptr; Rows::discard(); } const LevTrEntry& get_levtr() const { if (levtr == nullptr) // We prefetch levtr info for all IDs, so we do not need to hit the database here levtr = &(this->tr->levtr().lookup_cache(this->cur->id_levtr)); return *levtr; } }; struct BaseDataRows : public LevTrRows { using LevTrRows::LevTrRows; void enq(impl::Enq& enq) const; }; struct DataRows : public BaseDataRows { using BaseDataRows::BaseDataRows; int insert_cur_prio; /// Append or replace the last result according to priority. Returns false if the value has been ignored. bool add_to_best_results(const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var); void load(Tracer<>& trc, const DataQueryBuilder& qb); void load_best(Tracer<>& trc, const DataQueryBuilder& qb); }; struct SummaryRows : public LevTrRows { using LevTrRows::LevTrRows; void load(Tracer<>& trc, const SummaryQueryBuilder& qb); void enq(impl::Enq& enq) const; }; template struct ImplTraits { }; template<> struct ImplTraits { typedef dballe::CursorStation Interface; typedef db::CursorStation Parent; typedef StationRow Row; typedef StationRows Rows; }; template<> struct ImplTraits { typedef dballe::CursorStationData Interface; typedef db::CursorStationData Parent; typedef StationDataRow Row; typedef StationDataRows Rows; }; template<> struct ImplTraits { typedef dballe::CursorData Interface; typedef db::CursorData Parent; typedef DataRow Row; typedef DataRows Rows; }; template<> struct ImplTraits { typedef dballe::CursorSummary Interface; typedef db::CursorSummary Parent; typedef SummaryRow Row; typedef SummaryRows Rows; }; /** * Structure used to build and execute a query, and to iterate through the * results */ template struct Base : public ImplTraits::Parent { typedef typename ImplTraits::Row Row; typedef typename ImplTraits::Rows Rows; typedef typename ImplTraits::Interface Interface; Rows rows; Base(std::shared_ptr tr) : rows(tr) { } virtual ~Base() {} int remaining() const override; bool has_value() const { return !rows.at_start && rows.cur != rows.results.end(); } bool next() override { return rows.next(); } void discard() override; dballe::DBStation get_station() const override { return rows->station; } void enq(impl::Enq& enq) const override { return rows.enq(enq); } /** * Iterate the cursor until the end, returning the number of items. * * If dump is a FILE pointer, also dump the cursor values to it */ unsigned test_iterate(FILE* dump=0) override; /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { Impl* res = dynamic_cast(c.get()); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of cursor"); c.release(); return std::unique_ptr(res); } }; extern template class Base; extern template class Base; extern template class Base; extern template class Base; /// CursorStation implementation struct Stations : public Base { using Base::Base; DBValues get_values() const override; void remove() override; }; /// CursorStationData implementation struct StationData : public Base { bool with_attributes; StationData(DataQueryBuilder& qb, bool with_attributes); std::shared_ptr get_transaction() const override { return rows.tr; } wreport::Varcode get_varcode() const override { return rows->value.code(); } wreport::Var get_var() const override { return *rows->value; } int attr_reference_id() const override { return rows->value.data_id; } void query_attrs(std::function)> dest, bool force_read) override; void remove() override; }; /// CursorData implementation struct Data : public Base { bool with_attributes; Data(DataQueryBuilder& qb, bool with_attributes); std::shared_ptr get_transaction() const override { return rows.tr; } Datetime get_datetime() const override { return rows->datetime; } wreport::Varcode get_varcode() const override { return rows->value.code(); } wreport::Var get_var() const override { return *rows->value; } int attr_reference_id() const override { return rows->value.data_id; } Level get_level() const override { return rows.get_levtr().level; } Trange get_trange() const override { return rows.get_levtr().trange; } void query_attrs(std::function)> dest, bool force_read) override; void remove() override; }; /// CursorSummary implementation struct Summary : public Base { using Base::Base; DatetimeRange get_datetimerange() const override { return this->rows->dtrange; } Level get_level() const override { return rows.get_levtr().level; } Trange get_trange() const override { return rows.get_levtr().trange; } wreport::Varcode get_varcode() const override { return rows->code; } size_t get_count() const override { return rows->count; } void remove() override; }; std::unique_ptr run_station_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& query, bool explain); std::unique_ptr run_station_data_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& query, bool explain); std::unique_ptr run_data_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& query, bool explain); std::unique_ptr run_summary_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& query, bool explain); void run_delete_query(Tracer<>& trc, std::shared_ptr tr, const core::Query& query, bool station_vars, bool explain); } } } } #endif dballe-8.6/dballe/db/v7/data-test.cc0000644000175000017500000001045613554564112014110 00000000000000#include "dballe/db/tests.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/batch.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/station.h" #include "dballe/db/v7/levtr.h" #include "dballe/db/v7/data.h" #include "config.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct Fixture : EmptyTransactionFixture { using EmptyTransactionFixture::EmptyTransactionFixture; dballe::DBStation sde1; dballe::DBStation sde2; int lt1; int lt2; Fixture(const char* backend) : EmptyTransactionFixture(backend) { } void create_db() override { db::v7::Tracer<> trc; EmptyTransactionFixture::create_db(); sde1.report = "synop"; sde1.coords = Coords(4500000, 1100000); sde1.ident = "ciao"; sde2.report = "synop"; sde2.coords = Coords(4600000, 1200000); sde2.ident = nullptr; auto t = dynamic_pointer_cast(db->transaction()); // Insert a mobile station sde1.id = wcallchecked(t->station().insert_new(trc, sde1)); // Insert a fixed station sde2.id = wcallchecked(t->station().insert_new(trc, sde2)); // Insert a lev_tr lt1 = t->levtr().obtain_id(trc, db::v7::LevTrEntry(Level(1, 2, 0, 3), Trange(4, 5, 6))); // Insert another lev_tr lt2 = t->levtr().obtain_id(trc, db::v7::LevTrEntry(Level(2, 3, 1, 4), Trange(5, 6, 7))); t->commit(); } }; class Tests : public FixtureTestCase { using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg1("db_v7_data_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg3("db_v7_data_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg4("db_v7_data_mysql", "MYSQL"); #endif void Tests::register_tests() { add_method("insert", [](Fixture& f) { using namespace dballe::db::v7; Tracer<> trc; auto& da = f.tr->data(); // Insert a datum { Var var(varinfo(WR_VAR(0, 1, 2)), 123); std::vector vars; vars.emplace_back(f.lt1, &var); wassert(da.insert(trc, f.sde1.id, Datetime(2001, 2, 3, 4, 5, 6), vars, false)); wassert(actual(vars[0].id) == 1); } // Insert another datum { Var var(varinfo(WR_VAR(0, 1, 2)), 234); std::vector vars; vars.emplace_back(f.lt2, &var); wassert(da.insert(trc, f.sde2.id, Datetime(2002, 3, 4, 5, 6, 7), vars, false)); wassert(actual(vars[0].id) == 2); } // Reinsert the first datum: it should give an error, since V7 does // not check if old and new values are the same // Disabled because postgresql spits error also on stderr //{ // Var var(varinfo(WR_VAR(0, 1, 2)), 123); // std::vector vars; // vars.emplace_back(f.lt1, &var); // auto e = wassert_throws(std::runtime_error, da.insert(*f.tr, f.tr->station().get_id(*f.tr, f.sde1), Datetime(2001, 2, 3, 4, 5, 6), vars)); // wassert(actual(e.what()).matches("refusing to overwrite existing data")); //} // Reinsert the first datum, with a different value and overwrite: // it should find its ID and update it { Var var(varinfo(WR_VAR(0, 1, 2)), 125); std::vector vars; vars.emplace_back(1, f.lt1, &var); wassert(da.update(trc, vars, false)); wassert(actual(vars[0].id) == 1); } }); add_method("attrs", [](Fixture& f) { using namespace dballe::db::v7; Tracer<> trc; auto& da = f.tr->data(); // Insert a datum with attributes Var var(varinfo(WR_VAR(0, 1, 2)), 123); var.seta(newvar(WR_VAR(0, 33, 7), 50)); std::vector vars; vars.emplace_back(f.lt1, &var); wassert(da.insert(trc, f.sde1.id, Datetime(2001, 2, 3, 4, 5, 6), vars, true)); int id = vars[0].id; vector attrs; da.read_attrs(trc, id, [&](std::unique_ptr a) { attrs.emplace_back(*a); }); wassert(actual(attrs.size()) == 1); wassert(actual_varcode(attrs[0].code()) == WR_VAR(0, 33, 7)); wassert(actual(attrs[0]) == 50); }); } } dballe-8.6/dballe/db/v7/transaction.cc0000644000175000017500000002244013554564112014543 00000000000000#include "transaction.h" #include "db.h" #include "driver.h" #include "levtr.h" #include "cursor.h" #include "station.h" #include "repinfo.h" #include "batch.h" #include "trace.h" #include "dballe/core/query.h" #include "dballe/core/data.h" #include "dballe/sql/sql.h" #include #include using namespace wreport; using namespace std; namespace dballe { namespace db { namespace v7 { Transaction::Transaction(std::shared_ptr db, std::unique_ptr sql_transaction) : db(db), sql_transaction(sql_transaction.release()), batch(*this), trc(db->trace->trace_transaction()) { m_repinfo = db->driver().create_repinfo(*this).release(); m_station = db->driver().create_station(*this).release(); m_levtr = db->driver().create_levtr(*this).release(); m_station_data = db->driver().create_station_data(*this).release(); m_data = db->driver().create_data(*this).release(); } Transaction::~Transaction() { rollback_nothrow(); delete m_data; delete m_station_data; delete m_levtr; delete m_station; delete m_repinfo; delete sql_transaction; } v7::Repinfo& Transaction::repinfo() { return *m_repinfo; } v7::Station& Transaction::station() { return *m_station; } v7::LevTr& Transaction::levtr() { return *m_levtr; } v7::StationData& Transaction::station_data() { return *m_station_data; } v7::Data& Transaction::data() { return *m_data; } void Transaction::commit() { if (fired) return; sql_transaction->commit(); clear_cached_state(); fired = true; trc.done(); } void Transaction::rollback() { if (fired) return; sql_transaction->rollback(); clear_cached_state(); fired = true; trc.done(); } void Transaction::rollback_nothrow() noexcept { if (fired) return; sql_transaction->rollback_nothrow(); clear_cached_state(); fired = true; trc.done(); } void Transaction::clear_cached_state() { repinfo().read_cache(); levtr().clear_cache(); station_data().clear_cache(); data().clear_cache(); batch.clear(); } Transaction& Transaction::downcast(dballe::db::Transaction& transaction) { v7::Transaction* t = dynamic_cast(&transaction); assert(t); return *t; } void Transaction::remove_all() { auto trc = db->trace->trace_remove_all(); db->driver().remove_all_v7(); // TODO: pass trace step clear_cached_state(); } void Transaction::insert_station_data(dballe::Data& vals, const dballe::DBInsertOptions& opts) { Tracer<> trc(this->trc ? this->trc->trace_insert_station_data() : nullptr); core::Data& data = core::Data::downcast(vals); batch::Station* st = batch.get_station(trc, data.station, opts.can_add_stations); // Add all the variables we find batch::StationData& sd = st->get_station_data(trc); for (auto& i: data.values) sd.add(i.get(), opts.can_replace ? batch::UPDATE : batch::ERROR); // Perform changes batch.write_pending(trc); // Read the IDs from the results data.station.id = st->id; for (auto& v: data.values) { auto i = sd.ids_by_code.find(v.code()); if (i == sd.ids_by_code.end()) continue; v.data_id = i->id; } } void Transaction::insert_data(dballe::Data& vals, const dballe::DBInsertOptions& opts) { core::Data& data = core::Data::downcast(vals); if (data.values.empty()) throw error_notfound("no variables found in input record"); Tracer<> trc(this->trc ? this->trc->trace_insert_data() : nullptr); batch::Station* st = batch.get_station(trc, data.station, opts.can_add_stations); batch::MeasuredData& md = st->get_measured_data(trc, data.datetime); if (data.level.is_missing()) throw std::runtime_error("cannot access measured data with undefined level"); if (data.trange.is_missing()) throw std::runtime_error("cannot access measured data with undefined trange"); // Insert the lev_tr data, and get the ID int id_levtr = levtr().obtain_id(trc, LevTrEntry(data.level, data.trange)); // Add all the variables we find for (auto& i: data.values) md.add(id_levtr, i.get(), opts.can_replace ? batch::UPDATE : batch::ERROR); // Perform changes batch.write_pending(trc); // Read the IDs from the results data.station.id = st->id; for (auto& v: data.values) { auto i = md.ids_on_db.find(IdVarcode(id_levtr, v.code())); if (i == md.ids_on_db.end()) continue; v.data_id = i->id; } } void Transaction::remove_station_data(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_remove_station_data(query) : nullptr); cursor::run_delete_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), true, db->explain_queries); batch.clear(); } void Transaction::remove_data(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_remove_data(query) : nullptr); cursor::run_delete_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), false, db->explain_queries); batch.clear(); } void Transaction::remove_station_data_by_id(int id) { Tracer<> trc(this->trc ? this->trc->trace_remove_station_data_by_id(id) : nullptr); station_data().remove_by_id(trc, id); batch.clear(); } void Transaction::remove_data_by_id(int id) { Tracer<> trc(this->trc ? this->trc->trace_remove_data_by_id(id) : nullptr); data().remove_by_id(trc, id); batch.clear(); } std::unique_ptr Transaction::query_stations(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_query_stations(query) : nullptr); auto res = cursor::run_station_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), db->explain_queries); return res; } std::unique_ptr Transaction::query_station_data(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_query_station_data(query) : nullptr); auto res = cursor::run_station_data_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), db->explain_queries); return res; } std::unique_ptr Transaction::query_data(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_query_data(query) : nullptr); auto res = cursor::run_data_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), db->explain_queries); return res; } std::unique_ptr Transaction::query_summary(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_query_summary(query) : nullptr); auto res = cursor::run_summary_query(trc, dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), db->explain_queries); return res; } void Transaction::attr_query_station(int data_id, std::function)> dest) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_query_station") : nullptr); // Create the query auto& d = station_data(); d.read_attrs(trc, data_id, dest); } void Transaction::attr_query_data(int data_id, std::function)> dest) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_query_data") : nullptr); // Create the query auto& d = data(); d.read_attrs(trc, data_id, dest); } void Transaction::attr_insert_station(int data_id, const Values& attrs) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_insert_station") : nullptr); auto& d = station_data(); d.merge_attrs(trc, data_id, attrs); } void Transaction::attr_insert_data(int data_id, const Values& attrs) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_insert_data") : nullptr); auto& d = data(); d.merge_attrs(trc, data_id, attrs); } void Transaction::attr_remove_station(int data_id, const db::AttrList& attrs) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_remove_station") : nullptr); if (attrs.empty()) { // Delete all attributes char buf[64]; snprintf(buf, 64, "UPDATE station_data SET attrs=NULL WHERE id=%d", data_id); Tracer<> trc_upd(trc ? trc->trace_update(buf, 1) : nullptr); db->conn->execute(buf); } else { auto& d = station_data(); d.remove_attrs(trc, data_id, attrs); } } void Transaction::attr_remove_data(int data_id, const db::AttrList& attrs) { Tracer<> trc(this->trc ? this->trc->trace_func("attr_remove_data") : nullptr); if (attrs.empty()) { // Delete all attributes char buf[64]; snprintf(buf, 64, "UPDATE data SET attrs=NULL WHERE id=%d", data_id); Tracer<> trc_upd(trc ? trc->trace_update(buf, 1) : nullptr); db->conn->execute(buf); } else { auto& d = data(); d.remove_attrs(trc, data_id, attrs); } } void Transaction::update_repinfo(const char* repinfo_file, int* added, int* deleted, int* updated) { // TODO: tracing repinfo().update(repinfo_file, added, deleted, updated); } void Transaction::dump(FILE* out) { repinfo().dump(out); station().dump(out); levtr().dump(out); station_data().dump(out); data().dump(out); } void TestTransaction::commit() { throw std::runtime_error("commit attempted while forbidden during tests"); } } } } dballe-8.6/dballe/db/v7/batch.cc0000644000175000017500000002204613554564112013301 00000000000000#include "batch.h" #include "transaction.h" #include "station.h" #include namespace dballe { namespace db { namespace v7 { Batch::~Batch() { // Do not try to flush it, pending data may be lost unless write_pending is // called, and it's ok delete last_station; } void Batch::set_write_attrs(bool write_attrs) { if (this->write_attrs != write_attrs) { // Throw away pending data. It should not cause damage, since all // insert operations have their own write_pending at the end // write_pending(); clear(); } this->write_attrs = write_attrs; } bool Batch::have_station(const std::string& report, const Coords& coords, const Ident& ident) { return last_station && last_station->report == report && last_station->coords == coords && last_station->ident == ident; } void Batch::new_station(Tracer<>& trc, const std::string& report, const Coords& coords, const Ident& ident) { if (last_station) { last_station->write_pending(trc, write_attrs); delete last_station; last_station = nullptr; } last_station = new batch::Station(*this); last_station->report = report; last_station->coords = coords; last_station->ident = ident; } batch::Station* Batch::get_station(Tracer<>& trc, const dballe::DBStation& station, bool station_can_add) { v7::Station& st = transaction.station(); if (station.coords.is_missing()) { if (station.id == MISSING_INT) throw std::runtime_error("cannot use station information without both coordinates and ana_id"); if (last_station && last_station->id == station.id) return last_station; DBStation from_db = st.lookup(trc, station.id); new_station(trc, from_db.report, from_db.coords, from_db.ident); ++count_select_stations; last_station->id = station.id; } else { if (have_station(station.report, station.coords, station.ident)) return last_station; new_station(trc, station.report, station.coords, station.ident); ++count_select_stations; last_station->id = st.maybe_get_id(trc, *last_station); } if (last_station->id == MISSING_INT) { if (!station_can_add) throw wreport::error_notfound("station not found in the database"); last_station->is_new = true; last_station->station_data.loaded = true; } else { last_station->is_new = false; last_station->station_data.loaded = false; } return last_station; } batch::Station* Batch::get_station(Tracer<>& trc, const std::string& report, const Coords& coords, const Ident& ident) { if (have_station(report, coords, ident)) return last_station; v7::Station& st = transaction.station(); new_station(trc, report, coords, ident); last_station->id = st.maybe_get_id(trc, *last_station); ++count_select_stations; if (last_station->id == MISSING_INT) { last_station->is_new = true; last_station->station_data.loaded = true; } else { last_station->is_new = false; last_station->station_data.loaded = false; } return last_station; } void Batch::write_pending(Tracer<>& trc) { if (!last_station) return; last_station->write_pending(trc, write_attrs); } void Batch::clear() { delete last_station; last_station = nullptr; } void Batch::dump(FILE* out) const { fprintf(out, " * Batch wa:%d csst:%u cssd: %u, csd: %u\n", (int)write_attrs, count_select_stations, count_select_station_data, count_select_data); if (last_station) { fprintf(out, "Cached station:\n"); last_station->dump(out); } else { fprintf(out, "No cached station.\n"); } } namespace batch { void StationDatum::dump(FILE* out) const { fprintf(out, "%01d%02d%03d(%d): %s\n", WR_VAR_FXY(var->code()), (int)(var->code()), var->isset() ? var->enqc() : "(null)"); } void MeasuredDatum::dump(FILE* out) const { fprintf(out, "ltr:%d %01d%02d%03d(%d): %s\n", id_levtr, WR_VAR_FXY(var->code()), (int)(var->code()), var->isset() ? var->enqc() : "(null)"); } void StationData::add(const wreport::Var* var, UpdateMode on_conflict) { if (!loaded) throw std::runtime_error("StationData::add called without loading status from DB first"); auto in_db = ids_by_code.find(var->code()); if (in_db != ids_by_code.end()) { // Exists in the database switch (on_conflict) { case UPDATE: to_update.emplace_back(in_db->id, var); break; case IGNORE: break; case ERROR: throw wreport::error_consistency("refusing to overwrite existing data"); } } else { // Does not exist in the database to_insert.emplace_back(var); } } void StationData::write_pending(Tracer<>& trc, Transaction& tr, int station_id, bool with_attrs) { if (!to_insert.empty()) { auto& st = tr.station_data(); st.insert(trc, station_id, to_insert, with_attrs); for (const auto& v: to_insert) { auto cur = ids_by_code.find(v.var->code()); if (cur == ids_by_code.end()) ids_by_code.add(IdVarcode(v.id, v.var->code())); else cur->id = v.id; } } if (!to_update.empty()) { auto& st = tr.station_data(); st.update(trc, to_update, with_attrs); } to_insert.clear(); to_update.clear(); } void MeasuredData::add(int id_levtr, const wreport::Var* var, UpdateMode on_conflict) { auto in_db = ids_on_db.find(IdVarcode(id_levtr, var->code())); if (in_db != ids_on_db.end()) { // Exists in the database switch (on_conflict) { case UPDATE: to_update.emplace_back(in_db->id, id_levtr, var); break; case IGNORE: break; case ERROR: throw wreport::error_consistency("refusing to overwrite existing data"); } } else { // Does not exist in the database to_insert.emplace_back(id_levtr, var); } } void MeasuredData::write_pending(Tracer<>& trc, Transaction& tr, int station_id, bool with_attrs) { if (!to_insert.empty()) { auto& st = tr.data(); st.insert(trc, station_id, datetime, to_insert, with_attrs); for (const auto& v: to_insert) { auto cur = ids_on_db.find(IdVarcode(v.id_levtr, v.var->code())); if (cur == ids_on_db.end()) ids_on_db.add(MeasuredDataID(IdVarcode(v.id_levtr, v.var->code()), v.id)); else cur->id = v.id; } } if (!to_update.empty()) { auto& st = tr.data(); st.update(trc, to_update, with_attrs); } to_insert.clear(); to_update.clear(); } MeasuredDataVector::~MeasuredDataVector() { for (auto md: items) delete md; } StationData& Station::get_station_data(Tracer<>& trc) { if (!station_data.loaded) { v7::StationData& sd = batch.transaction.station_data(); sd.query(trc, id, [&](int data_id, wreport::Varcode code) { station_data.ids_by_code.add(IdVarcode(data_id, code)); }); station_data.loaded = true; ++batch.count_select_station_data; } return station_data; } MeasuredData& Station::get_measured_data(Tracer<>& trc, const Datetime& datetime) { if (datetime.is_missing()) throw std::runtime_error("cannot access measured data with undefined datetime"); auto mdi = measured_data.find(datetime); if (mdi != measured_data.end()) return **mdi; MeasuredData* md = measured_data.add(new MeasuredData(datetime)); if (!is_new) { v7::Data& d = batch.transaction.data(); d.query(trc, id, datetime, [&](int data_id, int id_levtr, wreport::Varcode code) { md->ids_on_db.add(MeasuredDataID(IdVarcode(id_levtr, code), data_id)); }); ++batch.count_select_data; } return *md; } void Station::write_pending(Tracer<>& trc, bool with_attrs) { if (id == MISSING_INT) id = batch.transaction.station().insert_new(trc, *this); station_data.write_pending(trc, batch.transaction, id, with_attrs); for (auto md: measured_data) md->write_pending(trc, batch.transaction, id, with_attrs); } void Station::dump(FILE* out) const { fprintf(out, "Station%s: ", is_new ? " (new)" : ""); print(out); } #if 0 void InsertStationVars::dump(FILE* out) const { fprintf(out, "ID station: %d\n", shared_context.station); for (unsigned i = 0; i < size(); ++i) { fprintf(out, "%3u/%3zd: ", i, size()); (*this)[i].dump(out); } } void InsertVars::dump(FILE* out) const { const auto& dt = shared_context.datetime; fprintf(out, "ID station: %d, datetime: %04d-%02d-%02d %02d:%02d:%02d\n", shared_context.station, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); for (unsigned i = 0; i < size(); ++i) { fprintf(out, "%3u/%3zd: ", i, size()); (*this)[i].dump(out); } } #endif } } } } dballe-8.6/dballe/db/v7/mysql/0000755000175000017500000000000013602152021013114 500000000000000dballe-8.6/dballe/db/v7/mysql/levtr.h0000644000175000017500000000202313554564112014354 00000000000000#ifndef DBALLE_DB_V7_MYSQL_LEVTRV7_H #define DBALLE_DB_V7_MYSQL_LEVTRV7_H #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace mysql { struct DB; /** * Precompiled queries to manipulate the lev_tr table */ struct MySQLLevTr : public v7::LevTr { protected: /** * DB connection. */ dballe::sql::MySQLConnection& conn; void _dump(std::function out) override; public: MySQLLevTr(v7::Transaction& tr, dballe::sql::MySQLConnection& conn); MySQLLevTr(const LevTr&) = delete; MySQLLevTr(const LevTr&&) = delete; MySQLLevTr& operator=(const MySQLLevTr&) = delete; ~MySQLLevTr(); void prefetch_ids(Tracer<>& trc, const std::set& ids) override; const LevTrEntry* lookup_id(Tracer<>& trc, int id) override; int obtain_id(Tracer<>& trc, const LevTrEntry& desc) override; }; } } } } #endif dballe-8.6/dballe/db/v7/mysql/station.h0000644000175000017500000000255113554564112014707 00000000000000#ifndef DBALLE_DB_V7_MYSQL_STATION_H #define DBALLE_DB_V7_MYSQL_STATION_H #include #include #include namespace wreport { struct Var; } namespace dballe { namespace db { namespace v7 { namespace mysql { /** * Precompiled queries to manipulate the station table */ class MySQLStation : public v7::Station { protected: /** * DB connection. */ dballe::sql::MySQLConnection& conn; void _dump(std::function out) override; public: MySQLStation(v7::Transaction& tr, dballe::sql::MySQLConnection& conn); ~MySQLStation(); MySQLStation(const MySQLStation&) = delete; MySQLStation(const MySQLStation&&) = delete; MySQLStation& operator=(const MySQLStation&) = delete; DBStation lookup(Tracer<>& trc, int id_station) override; int maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) override; int insert_new(Tracer<>& trc, const dballe::DBStation& desc) override; void get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) override; void add_station_vars(Tracer<>& trc, int id_station, DBValues& values) override; void run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function) override; }; } } } } #endif dballe-8.6/dballe/db/v7/mysql/driver.h0000644000175000017500000000155013554564112014517 00000000000000#ifndef DBALLE_DB_V7_MYSQL_DRIVER_H #define DBALLE_DB_V7_MYSQL_DRIVER_H #include #include namespace dballe { namespace db { namespace v7 { namespace mysql { struct Driver : public v7::Driver { dballe::sql::MySQLConnection& conn; Driver(dballe::sql::MySQLConnection& conn); virtual ~Driver(); std::unique_ptr create_repinfo(v7::Transaction& tr) override; std::unique_ptr create_station(v7::Transaction& tr) override; std::unique_ptr create_levtr(v7::Transaction& tr) override; std::unique_ptr create_station_data(v7::Transaction& tr) override; std::unique_ptr create_data(v7::Transaction& tr) override; void create_tables_v7() override; void delete_tables_v7() override; void vacuum_v7() override; }; } } } } #endif dballe-8.6/dballe/db/v7/mysql/repinfo.h0000644000175000017500000000230013554564112014660 00000000000000#ifndef DBALLE_DB_V7_MYSQL_REPINFO_H #define DBALLE_DB_V7_MYSQL_REPINFO_H #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace mysql { /** * Fast cached access to the repinfo table */ struct MySQLRepinfoV7 : public v7::Repinfo { /** * DB connection. The pointer is assumed always valid during the * lifetime of the object */ dballe::sql::MySQLConnection& conn; MySQLRepinfoV7(dballe::sql::MySQLConnection& conn); MySQLRepinfoV7(const MySQLRepinfoV7&) = delete; MySQLRepinfoV7(const MySQLRepinfoV7&&) = delete; virtual ~MySQLRepinfoV7(); MySQLRepinfoV7& operator=(const MySQLRepinfoV7&) = delete; void dump(FILE* out) override; protected: /// Return how many time this ID is used in the database int id_use_count(unsigned id, const char* name) override; void delete_entry(unsigned id) override; void update_entry(const v7::repinfo::Cache& entry) override; void insert_entry(const v7::repinfo::Cache& entry) override; void read_cache() override; void insert_auto_entry(const char* memo) override; }; } } } } #endif dballe-8.6/dballe/db/v7/mysql/repinfo.cc0000644000175000017500000000733413554564112015032 00000000000000#include "repinfo.h" #include "dballe/db/db.h" #include "dballe/sql/mysql.h" #include "dballe/sql/querybuf.h" using namespace wreport; using namespace std; using dballe::sql::MySQLConnection; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace mysql { MySQLRepinfoV7::MySQLRepinfoV7(MySQLConnection& conn) : Repinfo(conn), conn(conn) { read_cache(); } MySQLRepinfoV7::~MySQLRepinfoV7() { } void MySQLRepinfoV7::read_cache() { cache.clear(); memo_idx.clear(); auto res = conn.exec_store("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); while (auto row = res.fetch()) cache_append( row.as_int(0), row.as_cstring(1), row.as_cstring(2), row.as_int(3), row.as_cstring(4), row.as_int(5) ); // Rebuild the memo index as well rebuild_memo_idx(); } void MySQLRepinfoV7::insert_auto_entry(const char* memo) { unsigned id = conn.exec_store("SELECT MAX(id) FROM repinfo").expect_one_result().as_int(0); unsigned prio = conn.exec_store("SELECT MAX(prio) FROM repinfo").expect_one_result().as_int(0); ++id; ++prio; string escaped_memo = conn.escape(memo); Querybuf iq; iq.appendf(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES (%u, '%s', '%s', %u, '0', 255) )", id, escaped_memo.c_str(), escaped_memo.c_str(), prio); conn.exec_no_data(iq); } int MySQLRepinfoV7::id_use_count(unsigned id, const char* name) { char query[64]; snprintf(query, 64, "SELECT COUNT(1) FROM station WHERE rep=%d", id); return conn.exec_store(query).expect_one_result().as_int(0); } void MySQLRepinfoV7::delete_entry(unsigned id) { char query[64]; snprintf(query, 64, "DELETE FROM repinfo WHERE id=%d", id); conn.exec_no_data(query); } void MySQLRepinfoV7::update_entry(const v7::repinfo::Cache& entry) { Querybuf q; string escaped_memo = conn.escape(entry.new_memo); string escaped_desc = conn.escape(entry.new_desc); string escaped_descriptor = conn.escape(entry.new_descriptor); q.appendf(R"( UPDATE repinfo set memo='%s', description='%s', prio=%d, descriptor='%s', tablea=%u WHERE id=%u )", escaped_memo.c_str(), escaped_desc.c_str(), entry.new_prio, escaped_descriptor.c_str(), entry.new_tablea, entry.id); conn.exec_no_data(q); } void MySQLRepinfoV7::insert_entry(const v7::repinfo::Cache& entry) { Querybuf q; string escaped_memo = conn.escape(entry.new_memo); string escaped_desc = conn.escape(entry.new_desc); string escaped_descriptor = conn.escape(entry.new_descriptor); q.appendf(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES (%u, '%s', '%s', %d, '%s', %u) )", entry.id, escaped_memo.c_str(), escaped_desc.c_str(), entry.new_prio, escaped_descriptor.c_str(), entry.new_tablea); conn.exec_no_data(q); } void MySQLRepinfoV7::dump(FILE* out) { fprintf(out, "dump of table repinfo:\n"); fprintf(out, " id memo description prio desc tablea\n"); int count = 0; auto res = conn.exec_store("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); while (auto row = res.fetch()) { fprintf(out, " %4d %s %s %d %s %d\n", row.as_int(0), row.as_cstring(1), row.as_cstring(2), row.as_int(3), row.as_cstring(4), row.as_int(5)); ++count; } fprintf(out, "%d element%s in table repinfo\n", count, count != 1 ? "s" : ""); } } } } } dballe-8.6/dballe/db/v7/mysql/data.cc0000644000175000017500000003406213554564112014277 00000000000000#include "data.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/batch.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/db/v7/repinfo.h" #include "dballe/sql/mysql.h" #include "dballe/sql/querybuf.h" #include "dballe/values.h" #include "dballe/core/values.h" #include "dballe/core/varmatch.h" #include #include using namespace wreport; using namespace std; using dballe::sql::MySQLConnection; using dballe::sql::MySQLStatement; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace mysql { template class MySQLDataCommon; template class MySQLDataCommon; template MySQLDataCommon::MySQLDataCommon(v7::Transaction& tr, dballe::sql::MySQLConnection& conn) : Parent(tr), conn(conn) { } template MySQLDataCommon::~MySQLDataCommon() { } template void MySQLDataCommon::read_attrs(Tracer<>& trc, int id_data, std::function)> dest) { char query[128]; snprintf(query, 128, "SELECT attrs FROM %s WHERE id=%d", Parent::table_name, id_data); Tracer<> trc_sel(trc ? trc->trace_select(query) : nullptr); Values::decode( conn.exec_store(query).expect_one_result().as_blob(0), dest); if (trc_sel) trc_sel->add_row(); } template void MySQLDataCommon::write_attrs(Tracer<>& trc, int id_data, const Values& values) { vector encoded = values.encode(); string escaped = conn.escape(encoded); Querybuf qb; qb.appendf("UPDATE %s SET attrs=X'%s' WHERE id=%d", Parent::table_name, escaped.c_str(), id_data); Tracer<> trc_upd(trc ? trc->trace_update(qb, 1) : nullptr); conn.exec_no_data(qb); } template void MySQLDataCommon::remove_all_attrs(Tracer<>& trc, int id_data) { char query[128]; snprintf(query, 128, "UPDATE %s SET attrs=NULL WHERE id=%d", Parent::table_name, id_data); Tracer<> trc_upd(trc ? trc->trace_update(query, 1) : nullptr); conn.exec_no_data(query); } namespace { bool match_attrs(const Varmatch& match, const std::vector& attrs) { bool found = false; Values::decode(attrs, [&](std::unique_ptr var) { if (match(*var)) found = true; }); return found; } } template void MySQLDataCommon::remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) { if (qb.bind_in_ident) throw error_unimplemented("binding in MySQL driver is not implemented"); std::unique_ptr attr_filter; if (!qb.query.attr_filter.empty()) attr_filter = Varmatch::parse(qb.query.attr_filter); Querybuf dq(512); dq.appendf("DELETE FROM %s WHERE id IN (", Parent::table_name); dq.start_list(","); unsigned count = 0; Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); auto res = conn.exec_store(qb.sql_query); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); if (attr_filter.get() && !match_attrs(*attr_filter, row.as_blob(1))) return; // Note: if the query gets too long, we can split this in more DELETE // runs dq.append_list(row.as_cstring(0)); ++count; } dq.append(")"); if (count) { Tracer<> trc_del(trc ? trc->trace_delete(dq, count) : nullptr); conn.exec_no_data(dq); } } template void MySQLDataCommon::remove_by_id(Tracer<>& trc, int id) { char query[64]; snprintf(query, 64, "DELETE FROM %s WHERE id=%d", Parent::table_name, id); // Iterate all the data_id results, deleting the related data and attributes Tracer<> trc_sel(trc ? trc->trace_delete(query, 1) : nullptr); conn.exec_no_data(query); } template void MySQLDataCommon::update(Tracer<>& trc, std::vector& vars, bool with_attrs) { for (auto& v: vars) { string escaped_value = conn.escape(v.var->enqc()); Querybuf qb; if (with_attrs && v.var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v.var); string escaped_attrs = conn.escape(enc.buf); qb.appendf("UPDATE %s SET value='%s', attrs=X'%s' WHERE id=%d", Parent::table_name, escaped_value.c_str(), escaped_attrs.c_str(), v.id); } else qb.appendf("UPDATE %s SET value='%s', attrs=NULL WHERE id=%d", Parent::table_name, escaped_value.c_str(), v.id); Tracer<> trc_upd(trc ? trc->trace_update(qb, 1) : nullptr); conn.exec_no_data(qb); } } MySQLStationData::MySQLStationData(v7::Transaction& tr, MySQLConnection& conn) : MySQLDataCommon(tr, conn) { } void MySQLStationData::query(Tracer<>& trc, int id_station, std::function dest) { char strquery[128]; snprintf(strquery, 128, "SELECT id, code FROM station_data WHERE id_station=%d", id_station); Tracer<> trc_sel(trc ? trc->trace_select(strquery) : nullptr); auto res = conn.exec_store(strquery); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); int id = row.as_int(0); wreport::Varcode code = row.as_int(1); dest(id, code); } } void MySQLStationData::insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; Querybuf qb; string escaped_value = conn.escape(v->var->enqc()); if (with_attrs && v->var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v->var); string escaped_attrs = conn.escape(enc.buf); qb.appendf("INSERT INTO station_data (id_station, code, value, attrs) VALUES (%d, %d, '%s', X'%s')", id_station, (int)v->var->code(), escaped_value.c_str(), escaped_attrs.c_str()); } else qb.appendf("INSERT INTO station_data (id_station, code, value, attrs) VALUES (%d, %d, '%s', NULL)", id_station, (int)v->var->code(), escaped_value.c_str()); Tracer<> trc_ins(trc ? trc->trace_insert(qb, 1) : nullptr); conn.exec_no_data(qb); v->id = conn.get_last_insert_id(); } } void MySQLStationData::run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { if (qb.bind_in_ident) throw error_unimplemented("binding in MySQL driver is not implemented"); Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); dballe::DBStation station; conn.exec_use(qb.sql_query, [&](const sql::mysql::Row& row) { if (trc_sel) trc_sel->add_row(); wreport::Varcode code = row.as_int(5); const char* value = row.as_cstring(7); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(row.as_blob(8), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = row.as_int(0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(row.as_int(1)); station.coords.lat = row.as_int(2); station.coords.lon = row.as_int(3); if (row.isnull(4)) station.ident.clear(); else station.ident = row.as_string(4); } int id_data = row.as_int(6); dest(station, id_data, move(var)); }); } void MySQLStationData::dump(FILE* out) { StationDataDumper dumper(out); dumper.print_head(); auto res = conn.exec_store("SELECT id, id_station, code, value, attrs FROM station_data"); while (auto row = res.fetch()) { const char* val = row.isnull(3) ? nullptr : row.as_cstring(3); dumper.print_row(row.as_int(0), row.as_int(1), row.as_int(2), val, row.as_blob(4)); } dumper.print_tail(); } MySQLData::MySQLData(v7::Transaction& tr, MySQLConnection& conn) : MySQLDataCommon(tr, conn) { } void MySQLData::query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) { const auto& dt = datetime; char strquery[128]; snprintf(strquery, 128, "SELECT id, id_levtr, code FROM data WHERE id_station=%d AND datetime='%04d-%02d-%02d %02d:%02d:%02d'", id_station, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second); Tracer<> trc_sel(trc ? trc->trace_select(strquery) : nullptr); auto res = conn.exec_store(strquery); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); int id_levtr = row.as_int(1); wreport::Varcode code = row.as_int(2); int id = row.as_int(0); dest(id, id_levtr, code); } } void MySQLData::insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; Querybuf qb; const auto& dt = datetime; string escaped_value = conn.escape(v->var->enqc()); if (with_attrs && v->var->next_attr()) { core::value::Encoder enc; enc.append_attributes(*v->var); string escaped_attrs = conn.escape(enc.buf); qb.appendf("INSERT INTO data (id_station, id_levtr, datetime, code, value, attrs) VALUES (%d, %d, '%04d-%02d-%02d %02d:%02d:%02d', %d, '%s', X'%s')", id_station, v->id_levtr, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, (int)v->var->code(), escaped_value.c_str(), escaped_attrs.c_str()); } else qb.appendf("INSERT INTO data (id_station, id_levtr, datetime, code, value, attrs) VALUES (%d, %d, '%04d-%02d-%02d %02d:%02d:%02d', %d, '%s', NULL)", id_station, v->id_levtr, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, (int)v->var->code(), escaped_value.c_str()); Tracer<> trc_ins(trc ? trc->trace_insert(qb, 1) : nullptr); conn.exec_no_data(qb); v->id = conn.get_last_insert_id(); } } void MySQLData::run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { if (qb.bind_in_ident) throw error_unimplemented("binding in MySQL driver is not implemented"); Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); dballe::DBStation station; conn.exec_use(qb.sql_query, [&](const sql::mysql::Row& row) { if (trc_sel) trc_sel->add_row(); wreport::Varcode code = row.as_int(6); const char* value = row.as_cstring(9); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(row.as_blob(10), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = row.as_int(0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(row.as_int(1)); station.coords.lat = row.as_int(2); station.coords.lon = row.as_int(3); if (row.isnull(4)) station.ident.clear(); else station.ident = row.as_string(4); } int id_levtr = row.as_int(5); int id_data = row.as_int(7); Datetime datetime = row.as_datetime(8); dest(station, id_levtr, datetime, id_data, move(var)); }); } void MySQLData::run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function dest) { if (qb.bind_in_ident) throw error_unimplemented("binding in MySQL driver is not implemented"); Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); dballe::DBStation station; conn.exec_use(qb.sql_query, [&](const sql::mysql::Row& row) { if (trc_sel) trc_sel->add_row(); int id_station = row.as_int(0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(row.as_int(1)); station.coords.lat = row.as_int(2); station.coords.lon = row.as_int(3); if (row.isnull(4)) station.ident.clear(); else station.ident = row.as_string(4); } int id_levtr = row.as_int(5); wreport::Varcode code = row.as_int(6); size_t count = 0; DatetimeRange datetime; if (qb.select_summary_details) { count = row.as_int(7); datetime = DatetimeRange(row.as_datetime(8), row.as_datetime(9)); } dest(station, id_levtr, code, datetime, count); }); } void MySQLData::dump(FILE* out) { DataDumper dumper(out); dumper.print_head(); auto res = conn.exec_store("SELECT id, id_station, id_levtr, datetime, code, value, attrs FROM data"); while (auto row = res.fetch()) { const char* val = row.isnull(5) ? nullptr : row.as_cstring(5); dumper.print_row(row.as_int(0), row.as_int(1), row.as_int(2), row.as_datetime(3), row.as_int(4), val, row.as_blob(6)); } dumper.print_tail(); } } } } } dballe-8.6/dballe/db/v7/mysql/data.h0000644000175000017500000000712613554564112014142 00000000000000#ifndef DBALLE_DB_V7_MYSQL_DATA_H #define DBALLE_DB_V7_MYSQL_DATA_H #include #include #include namespace dballe { namespace db { namespace v7 { namespace mysql { struct DB; template class MySQLDataCommon : public Parent { protected: /// DB connection dballe::sql::MySQLConnection& conn; #if 0 /// Precompiled read attributes statement dballe::sql::MySQLStatement* read_attrs_stm = nullptr; /// Precompiled write attributes statement dballe::sql::MySQLStatement* write_attrs_stm = nullptr; /// Precompiled remove attributes statement dballe::sql::MySQLStatement* remove_attrs_stm = nullptr; /// Precompiled select statement dballe::sql::MySQLStatement* sstm = nullptr; /// Precompiled insert statement dballe::sql::MySQLStatement* istm = nullptr; /// Precompiled update statement dballe::sql::MySQLStatement* ustm = nullptr; #endif public: MySQLDataCommon(v7::Transaction& tr, dballe::sql::MySQLConnection& conn); MySQLDataCommon(const MySQLDataCommon&) = delete; MySQLDataCommon(const MySQLDataCommon&&) = delete; MySQLDataCommon& operator=(const MySQLDataCommon&) = delete; ~MySQLDataCommon(); void update(Tracer<>& trc, std::vector& vars, bool with_attrs) override; void read_attrs(Tracer<>& trc, int id_data, std::function)> dest) override; void write_attrs(Tracer<>& trc, int id_data, const Values& values) override; void remove_all_attrs(Tracer<>& trc, int id_data) override; void remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) override; void remove_by_id(Tracer<>& trc, int id) override; }; extern template class MySQLDataCommon; extern template class MySQLDataCommon; /** * Precompiled query to manipulate the station data table */ class MySQLStationData : public MySQLDataCommon { public: using MySQLDataCommon::MySQLDataCommon; MySQLStationData(v7::Transaction& tr, dballe::sql::MySQLConnection& conn); void query(Tracer<>& trc, int id_station, std::function dest) override; void insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) override; void run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void dump(FILE* out) override; void clear_cache() override {} }; /** * Precompiled query to manipulate the data table */ class MySQLData : public MySQLDataCommon { public: using MySQLDataCommon::MySQLDataCommon; MySQLData(v7::Transaction& tr, dballe::sql::MySQLConnection& conn); void query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) override; void insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) override; void run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function) override; void dump(FILE* out) override; void clear_cache() override {} }; } } } } #endif dballe-8.6/dballe/db/v7/mysql/levtr.cc0000644000175000017500000001062313554564112014517 00000000000000#include "levtr.h" #include "dballe/core/defs.h" #include "dballe/msg/msg.h" #include "dballe/sql/querybuf.h" #include "dballe/sql/mysql.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include #include #include using namespace wreport; using namespace std; using dballe::sql::MySQLConnection; using dballe::sql::mysql::Row; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace mysql { namespace { Level to_level(Row& row, int first_id=0) { return Level( row.as_int(first_id), row.as_int(first_id + 1), row.as_int(first_id + 2), row.as_int(first_id + 3)); } Trange to_trange(Row& row, int first_id=0) { return Trange( row.as_int(first_id), row.as_int(first_id + 1), row.as_int(first_id + 2)); } } MySQLLevTr::MySQLLevTr(v7::Transaction& tr, MySQLConnection& conn) : v7::LevTr(tr), conn(conn) { } MySQLLevTr::~MySQLLevTr() { } void MySQLLevTr::prefetch_ids(Tracer<>& trc, const std::set& ids) { if (ids.empty()) return; sql::Querybuf qb; if (ids.size() < 100) { qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id IN ("); qb.start_list(","); for (auto id: ids) qb.append_listf("%d", id); qb.append(")"); } else qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr"); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec_store(qb); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); cache.insert(unique_ptr(new LevTrEntry( row.as_int(0), Level(row.as_int(1), row.as_int(2), row.as_int(3), row.as_int(4)), Trange(row.as_int(5), row.as_int(6), row.as_int(7))))); } } const LevTrEntry* MySQLLevTr::lookup_id(Tracer<>& trc, int id) { const LevTrEntry* res = cache.find_entry(id); if (res) return res; char query[128]; snprintf(query, 128, "SELECT ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id=%d", id); Tracer<> trc_sel(trc ? trc->trace_select(query) : nullptr); auto qres = conn.exec_store(query); while (auto row = qres.fetch()) { if (trc_sel) trc_sel->add_row(); std::unique_ptr e(new LevTrEntry); e->id = id; e->level.ltype1 = row.as_int(0); e->level.l1 = row.as_int(1); e->level.ltype2 = row.as_int(2); e->level.l2 = row.as_int(3); e->trange.pind = row.as_int(4); e->trange.p1 = row.as_int(5); e->trange.p2 = row.as_int(6); res = cache.insert(move(e)); } if (!res) error_notfound::throwf("levtr with id %d not found in the database", id); return res; } int MySQLLevTr::obtain_id(Tracer<>& trc, const LevTrEntry& desc) { int id = cache.find_id(desc); if (id != MISSING_INT) return id; char query[512]; snprintf(query, 512, R"( SELECT id FROM levtr WHERE ltype1=%d AND l1=%d AND ltype2=%d AND l2=%d AND pind=%d AND p1=%d AND p2=%d )", desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); // If there is an existing record, use its ID and don't do an INSERT Tracer<> trc_oid(trc ? trc->trace_select(query) : nullptr); auto qres = conn.exec_store(query); while (auto row = qres.fetch()) { if (trc_oid) trc_oid->add_row(); id = row.as_int(0); } if (id != MISSING_INT) { cache.insert(desc, id); return id; } // Not found in the database, insert a new one snprintf(query, 512, "INSERT INTO levtr (ltype1, l1, ltype2, l2, pind, p1, p2) VALUES (%d, %d, %d, %d, %d, %d, %d)", desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); trc_oid.reset(trc ? trc->trace_insert(query, 1) : nullptr); conn.exec_no_data(query); id = conn.get_last_insert_id(); cache.insert(desc, id); return id; } void MySQLLevTr::_dump(std::function out) { auto res = conn.exec_store("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr ORDER BY ID"); while (auto row = res.fetch()) out(row.as_int(0), to_level(row, 1), to_trange(row, 5)); } } } } } dballe-8.6/dballe/db/v7/mysql/station.cc0000644000175000017500000001705213554564112015047 00000000000000#include "station.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/sql/mysql.h" #include "dballe/sql/querybuf.h" #include "dballe/core/var.h" #include "dballe/values.h" #include using namespace wreport; using namespace dballe::db; using namespace std; using dballe::sql::MySQLConnection; using dballe::sql::MySQLStatement; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace mysql { MySQLStation::MySQLStation(v7::Transaction& tr, MySQLConnection& conn) : v7::Station(tr), conn(conn) { } MySQLStation::~MySQLStation() { } DBStation MySQLStation::lookup(Tracer<>& trc, int id_station) { Querybuf qb; qb.appendf("SELECT rep, lat, lon, ident FROM station WHERE id=%d", id_station); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec_store(qb); if (trc_sel) trc_sel->add_row(res.rowcount()); switch (res.rowcount()) { case 0: { stringstream msg; msg << "Station with id " << id_station << " not found"; throw std::runtime_error(msg.str()); } case 1: { auto row = res.fetch(); DBStation station; station.id = id_station; station.report = tr.repinfo().get_rep_memo(row.as_int(0)); station.coords.lat = row.as_int(1); station.coords.lon = row.as_int(2); if (row.isnull(3)) station.ident.clear(); else station.ident = row.as_string(3); return station; } default: error_consistency::throwf("select station data query returned %u results", res.rowcount()); } } int MySQLStation::maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) { int rep = tr.repinfo().obtain_id(st.report.c_str()); Querybuf qb; if (st.ident.get()) { string escaped_ident = conn.escape(st.ident.get()); qb.appendf("SELECT id FROM station WHERE rep=%d AND lat=%d AND lon=%d AND ident='%s'", rep, st.coords.lat, st.coords.lon, escaped_ident.c_str()); } else { qb.appendf("SELECT id FROM station WHERE rep=%d AND lat=%d AND lon=%d AND ident IS NULL", rep, st.coords.lat, st.coords.lon); } Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec_store(qb); if (trc_sel) trc_sel->add_row(res.rowcount()); switch (res.rowcount()) { case 0: return MISSING_INT; case 1: return res.fetch().as_int(0); default: error_consistency::throwf("select station ID query returned %u results", res.rowcount()); } } int MySQLStation::insert_new(Tracer<>& trc, const dballe::DBStation& desc) { // If no station was found, insert a new one int rep = tr.repinfo().get_id(desc.report.c_str()); Querybuf qb; if (desc.ident.get()) { string escaped_ident = conn.escape(desc.ident.get()); qb.appendf(R"( INSERT INTO station (rep, lat, lon, ident) VALUES (%d, %d, %d, '%s') )", rep, desc.coords.lat, desc.coords.lon, escaped_ident.c_str()); } else { qb.appendf(R"( INSERT INTO station (rep, lat, lon, ident) VALUES (%d, %d, %d, NULL) )", rep, desc.coords.lat, desc.coords.lon); } Tracer<> trc_ins(trc ? trc->trace_insert(qb, 1) : nullptr); conn.exec_no_data(qb); return conn.get_last_insert_id(); } #if 0 // See http://mikefenwick.com/blog/insert-into-database-or-return-id-of-duplicate-row-in-mysql/ // // It would be nice to use this, BUT at every failed insert the // auto_increment sequence is updated, so every lookup causes a hole in the // sequence. // // Since the auto_increment sequence blocks all inserts once it runs out of // numbers, we cannot afford to eat it away for every ID lookup. // // http://stackoverflow.com/questions/2615417/what-happens-when-auto-increment-on-integer-column-reaches-the-max-value-in-data // Just in case there's any question, the AUTO_INCREMENT field /DOES NOT // WRAP/. Once you hit the limit for the field size, INSERTs generate an // error. (As per Jeremy Cole) Querybuf qb; if (ident) { string escaped_ident = conn.escape(ident); qb.appendf(R"( INSERT INTO station (lat, lon, ident) VALUES (%d, %d, '%s') ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id) )", lat, lon, escaped_ident.c_str()); } else { qb.appendf(R"( INSERT INTO station (lat, lon, ident) VALUES (%d, %d, NULL) ON DUPLICATE KEY UPDATE id=LAST_INSERT_ID(id) )", lat, lon); } conn.exec_no_data(qb); if (inserted) *inserted = mysql_affected_rows(conn) > 0; return conn.get_last_insert_id(); #endif void MySQLStation::get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) { // Perform the query Querybuf qb; qb.appendf(R"( SELECT d.code, d.value, d.attrs FROM station_data d WHERE d.id_station=%d ORDER BY d.code )", id_station); TRACE("get_station_vars Performing query: %s\n", qb.c_str()); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec_store(qb); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); Varcode code = row.as_int(0); TRACE("get_station_vars Got %d%02d%03d %s\n", WR_VAR_FXY(code), row.as_cstring(1)); unique_ptr var = newvar(code, row.as_cstring(1)); if (!row.isnull(2)) { TRACE("get_station_vars add attributes\n"); DBValues::decode(row.as_blob(2), [&](unique_ptr a) { var->seta(move(a)); }); } dest(move(var)); } } void MySQLStation::add_station_vars(Tracer<>& trc, int id_station, DBValues& values) { Querybuf qb; qb.appendf(R"( SELECT d.code, d.value FROM station_data d WHERE d.id_station=%d )", id_station); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto res = conn.exec_store(qb); while (auto row = res.fetch()) { if (trc_sel) trc_sel->add_row(); values.set(newvar((wreport::Varcode)row.as_int(0), row.as_cstring(1))); } } void MySQLStation::run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function dest) { if (qb.bind_in_ident) throw error_unimplemented("binding in MySQL driver is not implemented"); Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); dballe::DBStation station; conn.exec_use(qb.sql_query, [&](const sql::mysql::Row& row) { if (trc_sel) trc_sel->add_row(); station.id = row.as_int(0); station.report = tr.repinfo().get_rep_memo(row.as_int(1)); station.coords.lat = row.as_int(2); station.coords.lon = row.as_int(3); if (row.isnull(4)) station.ident.clear(); else station.ident = row.as_string(4); dest(station); }); } void MySQLStation::_dump(std::function out) { auto res = conn.exec_store("SELECT id, rep, lat, lon, ident FROM station"); while (auto row = res.fetch()) { const char* ident = row.isnull(4) ? nullptr : row.as_cstring(4); out(row.as_int(0), row.as_int(1), Coords(row.as_int(2), row.as_int(3)), ident); } } } } } } dballe-8.6/dballe/db/v7/mysql/driver.cc0000644000175000017500000001104513554573614014664 00000000000000#include "driver.h" #include "repinfo.h" #include "station.h" #include "levtr.h" #include "data.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/sql/mysql.h" #include "dballe/var.h" #include #include using namespace std; using namespace wreport; using dballe::sql::MySQLConnection; using dballe::sql::MySQLStatement; using dballe::sql::Transaction; using dballe::sql::Querybuf; using dballe::sql::mysql::Row; namespace dballe { namespace db { namespace v7 { namespace mysql { Driver::Driver(MySQLConnection& conn) : v7::Driver(conn), conn(conn) { } Driver::~Driver() { } std::unique_ptr Driver::create_repinfo(v7::Transaction& tr) { return unique_ptr(new MySQLRepinfoV7(conn)); } std::unique_ptr Driver::create_station(v7::Transaction& tr) { return unique_ptr(new MySQLStation(tr, conn)); } std::unique_ptr Driver::create_levtr(v7::Transaction& tr) { return unique_ptr(new MySQLLevTr(tr, conn)); } std::unique_ptr Driver::create_station_data(v7::Transaction& tr) { return unique_ptr(new MySQLStationData(tr, conn)); } std::unique_ptr Driver::create_data(v7::Transaction& tr) { return unique_ptr(new MySQLData(tr, conn)); } // Extra options needed to fix MySQL's defaults. See: #153 #define DBA_MYSQL_DEFAULT_TABLE_OPTIONS "CHARACTER SET = utf8mb4, COLLATE = utf8mb4_bin, ENGINE = InnoDB" void Driver::create_tables_v7() { conn.exec_no_data(R"( CREATE TABLE repinfo ( id SMALLINT PRIMARY KEY, memo VARCHAR(20) NOT NULL, description VARCHAR(255) NOT NULL, prio INTEGER NOT NULL, descriptor CHAR(6) NOT NULL, tablea INTEGER NOT NULL, UNIQUE INDEX (prio), UNIQUE INDEX (memo) ) )" DBA_MYSQL_DEFAULT_TABLE_OPTIONS); conn.exec_no_data(R"( CREATE TABLE station ( id INTEGER auto_increment PRIMARY KEY, rep INTEGER NOT NULL REFERENCES repinfo (id) ON DELETE CASCADE, lat INTEGER NOT NULL, lon INTEGER NOT NULL, ident CHAR(64), UNIQUE INDEX(rep, lat, lon, ident(8)), INDEX(rep), INDEX(lon) ) )" DBA_MYSQL_DEFAULT_TABLE_OPTIONS); conn.exec_no_data(R"( CREATE TABLE levtr ( id INTEGER auto_increment PRIMARY KEY, ltype1 INTEGER NOT NULL, l1 INTEGER NOT NULL, ltype2 INTEGER NOT NULL, l2 INTEGER NOT NULL, pind INTEGER NOT NULL, p1 INTEGER NOT NULL, p2 INTEGER NOT NULL, UNIQUE INDEX (ltype1, l1, ltype2, l2, pind, p1, p2) ) )" DBA_MYSQL_DEFAULT_TABLE_OPTIONS); conn.exec_no_data(R"( CREATE TABLE station_data ( id INTEGER auto_increment PRIMARY KEY, id_station INTEGER NOT NULL REFERENCES station (id) ON DELETE CASCADE, code SMALLINT NOT NULL, value VARCHAR(255) NOT NULL, attrs BLOB, UNIQUE INDEX(id_station, code) ) )" DBA_MYSQL_DEFAULT_TABLE_OPTIONS); conn.exec_no_data(R"( CREATE TABLE data ( id INTEGER auto_increment PRIMARY KEY, id_station INTEGER NOT NULL, id_levtr INTEGER NOT NULL, datetime DATETIME NOT NULL, code SMALLINT NOT NULL, value VARCHAR(255) NOT NULL, attrs BLOB, UNIQUE INDEX(id_station, datetime, id_levtr, code), INDEX(id_levtr) ) )" DBA_MYSQL_DEFAULT_TABLE_OPTIONS); conn.set_setting("version", "V7"); } void Driver::delete_tables_v7() { conn.drop_table_if_exists("data"); conn.drop_table_if_exists("station_data"); conn.drop_table_if_exists("levtr"); conn.drop_table_if_exists("repinfo"); conn.drop_table_if_exists("station"); conn.drop_settings(); } void Driver::vacuum_v7() { conn.exec_no_data("DELETE ltr FROM levtr ltr LEFT JOIN data d ON d.id_levtr=ltr.id WHERE d.id_levtr IS NULL"); conn.exec_no_data(R"( DELETE sd FROM station_data sd LEFT JOIN data dd ON sd.id_station = dd.id_station WHERE dd.id IS NULL )"); conn.exec_no_data("DELETE s FROM station s LEFT JOIN data d ON d.id_station = s.id WHERE d.id IS NULL"); } } } } } dballe-8.6/dballe/db/v7/cursor-access.in.cc0000644000175000017500000001712313554573614015410 00000000000000#include "cursor.h" #include "repinfo.h" #include "levtr.h" #include using namespace wreport; namespace dballe { namespace db { namespace v7 { namespace cursor { /* * Stations */ void StationRows::enq(impl::Enq& enq) const { if (enq.search_b_values(values())) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": enq.set_int(get_priority()); case "rep_memo": enq.set_string(cur->station.report); case "report": enq.set_string(cur->station.report); case "ana_id": enq.set_dballe_int(cur->station.id); case "mobile": enq.set_bool(!cur->station.ident.is_missing()); case "ident": enq.set_ident(cur->station.ident); case "lat": enq.set_lat(cur->station.coords.lat); case "lon": enq.set_lon(cur->station.coords.lon); case "coords": enq.set_coords(cur->station.coords); case "station": enq.set_station(cur->station); default: enq.search_alias_values(values()); } } /* * StationData */ void StationDataRows::enq(impl::Enq& enq) const { if (enq.search_b_value(cur->value)) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": enq.set_int(get_priority()); case "rep_memo": enq.set_string(cur->station.report); case "report": enq.set_string(cur->station.report); case "ana_id": enq.set_dballe_int(cur->station.id); case "mobile": enq.set_bool(!cur->station.ident.is_missing()); case "ident": enq.set_ident(cur->station.ident); case "lat": enq.set_lat(cur->station.coords.lat); case "lon": enq.set_lon(cur->station.coords.lon); case "coords": enq.set_coords(cur->station.coords); case "station": enq.set_station(cur->station); case "var": enq.set_varcode(cur->value.code()); case "variable": enq.set_var(cur->value.get()); case "attrs": enq.set_attrs(cur->value.get()); case "context_id": enq.set_dballe_int(cur->value.data_id); default: enq.search_alias_value(cur->value); } } /* * Data */ void BaseDataRows::enq(impl::Enq& enq) const { if (enq.search_b_value(cur->value)) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": enq.set_int(get_priority()); case "rep_memo": enq.set_string(cur->station.report); case "report": enq.set_string(cur->station.report); case "ana_id": enq.set_dballe_int(cur->station.id); case "mobile": enq.set_bool(!cur->station.ident.is_missing()); case "ident": enq.set_ident(cur->station.ident); case "lat": enq.set_lat(cur->station.coords.lat); case "lon": enq.set_lon(cur->station.coords.lon); case "coords": enq.set_coords(cur->station.coords); case "station": enq.set_station(cur->station); case "datetime": enq.set_datetime(cur->datetime); case "year": enq.set_int(cur->datetime.year); case "month": enq.set_int(cur->datetime.month); case "day": enq.set_int(cur->datetime.day); case "hour": enq.set_int(cur->datetime.hour); case "min": enq.set_int(cur->datetime.minute); case "sec": enq.set_int(cur->datetime.second); case "level": enq.set_level(get_levtr().level); case "leveltype1": enq.set_dballe_int(get_levtr().level.ltype1); case "l1": enq.set_dballe_int(get_levtr().level.l1); case "leveltype2": enq.set_dballe_int(get_levtr().level.ltype2); case "l2": enq.set_dballe_int(get_levtr().level.l2); case "trange": enq.set_trange(get_levtr().trange); case "pindicator": enq.set_dballe_int(get_levtr().trange.pind); case "p1": enq.set_dballe_int(get_levtr().trange.p1); case "p2": enq.set_dballe_int(get_levtr().trange.p2); case "var": enq.set_varcode(cur->value.code()); case "variable": enq.set_var(cur->value.get()); case "attrs": enq.set_attrs(cur->value.get()); case "context_id": enq.set_dballe_int(cur->value.data_id); default: enq.search_alias_value(cur->value); } } /* * Summary */ void SummaryRows::enq(impl::Enq& enq) const { const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": enq.set_int(get_priority()); case "rep_memo": enq.set_string(cur->station.report); case "report": enq.set_string(cur->station.report); case "ana_id": enq.set_dballe_int(cur->station.id); case "mobile": enq.set_bool(!cur->station.ident.is_missing()); case "ident": enq.set_ident(cur->station.ident); case "lat": enq.set_lat(cur->station.coords.lat); case "lon": enq.set_lon(cur->station.coords.lon); case "coords": enq.set_coords(cur->station.coords); case "station": enq.set_station(cur->station); case "datetimemax": if (cur->dtrange.is_missing()) return; else enq.set_datetime(cur->dtrange.max); case "datetimemin": if (cur->dtrange.is_missing()) return; else enq.set_datetime(cur->dtrange.min); case "yearmax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.year); case "yearmin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.year); case "monthmax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.month); case "monthmin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.month); case "daymax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.day); case "daymin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.day); case "hourmax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.hour); case "hourmin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.hour); case "minumax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.minute); case "minumin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.minute); case "secmax": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.max.second); case "secmin": if (cur->dtrange.is_missing()) return; else enq.set_int(cur->dtrange.min.second); case "level": enq.set_level(get_levtr().level); case "leveltype1": enq.set_dballe_int(get_levtr().level.ltype1); case "l1": enq.set_dballe_int(get_levtr().level.l1); case "leveltype2": enq.set_dballe_int(get_levtr().level.ltype2); case "l2": enq.set_dballe_int(get_levtr().level.l2); case "trange": enq.set_trange(get_levtr().trange); case "pindicator": enq.set_dballe_int(get_levtr().trange.pind); case "p1": enq.set_dballe_int(get_levtr().trange.p1); case "p2": enq.set_dballe_int(get_levtr().trange.p2); case "var": enq.set_varcode(cur->code); case "context_id": enq.set_int(cur->count); case "count": enq.set_int(cur->count); default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } } } } } dballe-8.6/dballe/db/v7/driver.h0000644000175000017500000000401613554564112013352 00000000000000#ifndef DBALLE_DB_V7_DRIVER_H #define DBALLE_DB_V7_DRIVER_H #include #include #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { struct Driver { public: sql::Connection& connection; Driver(sql::Connection& connection); virtual ~Driver(); /// Precompiled queries to manipulate the repinfo table virtual std::unique_ptr create_repinfo(v7::Transaction& tr) = 0; /// Precompiled queries to manipulate the station table virtual std::unique_ptr create_station(v7::Transaction& tr) = 0; /// Precompiled queries to manipulate the levtr table virtual std::unique_ptr create_levtr(v7::Transaction& tr) = 0; /// Precompiled queries to manipulate the data table virtual std::unique_ptr create_station_data(v7::Transaction& tr) = 0; /// Precompiled queries to manipulate the data table virtual std::unique_ptr create_data(v7::Transaction& tr) = 0; /// Create all missing tables for a DB with the given format void create_tables(db::Format format); /// Create all missing tables for V7 databases virtual void create_tables_v7() = 0; /// Delete all existing tables for a DB with the given format void delete_tables(db::Format format); /// Delete all existing tables for V7 databases virtual void delete_tables_v7() = 0; /// Empty all tables for a DB with the given format void remove_all(db::Format format); /// Empty all tables for V7 databases, assuming that they exist, without touching the repinfo table virtual void remove_all_v7(); /// Perform database cleanup/maintenance on v7 databases virtual void vacuum_v7() = 0; /// Create a Driver for this connection static std::unique_ptr create(dballe::sql::Connection& conn); }; } } } #endif dballe-8.6/dballe/db/v7/db.h0000644000175000017500000000414213554564112012444 00000000000000#ifndef DBA_DB_V7_H #define DBA_DB_V7_H #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { /** * DB-ALLe database connection for database format V7 */ class DB : public dballe::db::DB { public: /// Database connection dballe::sql::Connection* conn; /// Database query tracing Trace* trace = nullptr; /// True if we print an EXPLAIN trace of all queries to stderr bool explain_queries = false; protected: /// SQL driver backend v7::Driver* m_driver; void init_after_connect(); public: DB(std::unique_ptr conn); virtual ~DB(); db::Format format() const { return Format::V7; } /// Access the backend DB driver v7::Driver& driver(); std::shared_ptr transaction(bool readonly=false) override; std::shared_ptr test_transaction(bool readonly=false) override; void disappear(); /** * Reset the database, removing all existing DBALLE tables and re-creating them * empty. * * @param repinfo_file * The name of the CSV file with the report type information data to load. * The file is in CSV format with 6 columns: report code, mnemonic id, * description, priority, descriptor, table A category. * If repinfo_file is NULL, then the default of /etc/dballe/repinfo.csv is * used. */ void reset(const char* repinfo_file = 0); /** * Delete all the DB-ALLe tables from the database. */ void delete_tables(); /** * Remove orphan values from the database. * * Orphan values are currently: * \li lev_tr values for which no data exists * \li station values for which no lev_tr exists * * Depending on database size, this routine can take a few minutes to execute. */ void vacuum(); friend class dballe::DB; friend class dballe::db::v7::Transaction; }; } } } #endif dballe-8.6/dballe/db/v7/repinfo.h0000644000175000017500000001227213554564112013524 00000000000000#ifndef DBALLE_DB_V7_REPINFO_H #define DBALLE_DB_V7_REPINFO_H /** @file * @ingroup db * * Repinfo table management used by the db module. */ #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace repinfo { /** repinfo cache entry */ struct Cache { /// Report code unsigned id; /** Report name */ std::string memo; /** Report description */ std::string desc; /// Report priority int prio; /** Report descriptor (currently unused) */ std::string descriptor; /// Report A table value (currently unused) unsigned tablea; /** New report name used when updating the repinfo table */ std::string new_memo; /** New report description used when updating the repinfo table */ std::string new_desc; /// New report priority used when updating the repinfo table int new_prio; /** New report descriptor used when updating the repinfo table */ std::string new_descriptor; /// New report A table value used when updating the repinfo table unsigned new_tablea; Cache(int id, const std::string& memo, const std::string& desc, int prio, const std::string& descriptor, int tablea); void make_new(); void dump(FILE* out) const; }; /** reverse rep_memo -> rep_cod cache entry */ struct Memoidx { /** Report name */ std::string memo; /** Report code */ int id; bool operator<(const Memoidx& memo) const; }; } /// Fast cached access to the repinfo table struct Repinfo { dballe::sql::Connection& conn; Repinfo(dballe::sql::Connection& conn); virtual ~Repinfo() {} //static std::unique_ptr create(Connection& conn); /// Get the rep_memo for a given ID; throws if id is not valud // FIXME: use std::string? const char* get_rep_memo(int id); /// Get the ID for a given rep_memo; returns -1 if rep_memo is not valid int get_id(const char* rep_memo); /// Get the priority for a given ID; returns INT_MAX if id is not valid int get_priority(const std::string& report); /** * Update the report type information in the database using the data from the * given file. * * @param deffile * Pathname of the file to use for the update. The NULL value is accepted * and means to use the default configure repinfo.csv file. * @retval added * Number of entries that have been added during the update. * @retval deleted * Number of entries that have been deleted during the update. * @retval updated * Number of entries that have been updated during the update. */ void update(const char* deffile, int* added, int* deleted, int* updated); /** * Get a mapping between rep_memo and their priorities */ std::map get_priorities(); /** * Return a vector of IDs matching the priority constraints in the given record. */ std::vector ids_by_prio(const core::Query& rec); /** * Get the id of a repinfo entry given its name. * * It creates a new entry if the memo is missing from the database. * * @param memo * The name to query * @return * The resulting id. */ int obtain_id(const char* memo); /// Dump the entire contents of the database to an output stream virtual void dump(FILE* out) = 0; /** Reread the repinfo cache from the database * FIXME: needed when rolling back a transaction, won't be needed anymore * when repinfo is moved to Transaction instead of db */ virtual void read_cache() = 0; protected: /** Cache of table entries */ std::vector cache; /** rep_memo -> rep_cod reverse index */ mutable std::vector memo_idx; /// Get a Cache entry by database ID const repinfo::Cache* get_by_id(unsigned id) const; /// Get a Cache entry by report name const repinfo::Cache* get_by_memo(const char* memo) const; /// Lookup a cache index by database ID. Returns -1 if not found int cache_find_by_id(unsigned id) const; /// Lookup a cache index by report name. Returns -1 if not found int cache_find_by_memo(const char* memo) const; /// Append an entry to the cache void cache_append(unsigned id, const char* memo, const char* desc, int prio, const char* descriptor, int tablea); /// Rebuild the memo_idx cache void rebuild_memo_idx() const; /// Read cache entries from a repinfo file on disk std::vector read_repinfo_file(const char* deffile); /// Return how many time this ID is used in the database virtual int id_use_count(unsigned id, const char* name) = 0; /// Delete a repinfo entry virtual void delete_entry(unsigned id) = 0; /// Update an entry using the new_* fields of \a entry virtual void update_entry(const repinfo::Cache& entry) = 0; /// Insert an entry using the new_* fields of \a entry virtual void insert_entry(const repinfo::Cache& entry) = 0; /// Create an automatic entry for a missing memo, and insert it in the database virtual void insert_auto_entry(const char* memo) = 0; }; } } } #endif dballe-8.6/dballe/db/v7/batch-test.cc0000644000175000017500000002132113554564112014251 00000000000000#include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/levtr.h" #include "dballe/var.h" #include "batch.h" #include "config.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct Fixture : EmptyTransactionFixture { using EmptyTransactionFixture::EmptyTransactionFixture; }; class Tests : public FixtureTestCase { using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg1("db_v7_batch_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg3("db_v7_batch_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg4("db_v7_batch_mysql", "MYSQL"); #endif void Tests::register_tests() { add_method("empty", [](Fixture& f) { using namespace dballe::db::v7; db::v7::Tracer<> trc; Batch& batch = f.tr->batch; batch::Station* station = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), Ident())); wassert(actual(station->report) == "synop"); wassert(actual(station->id) == MISSING_INT); wassert(actual(station->coords) == Coords(11.0, 45.0)); wassert_true(station->ident.is_missing()); wassert_true(station->is_new); wassert_true(station->station_data.ids_by_code.empty()); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); station = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), "AB123")); wassert(actual(station->report) == "synop"); wassert(actual(station->id) == MISSING_INT); wassert(actual(station->coords) == Coords(11.0, 45.0)); wassert_false(station->ident.is_missing()); wassert(actual(station->ident) == "AB123"); wassert_true(station->is_new); wassert_true(station->station_data.ids_by_code.empty()); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); }); add_method("reuse", [](Fixture& f) { using namespace dballe::db::v7; db::v7::Tracer<> trc; Batch& batch = f.tr->batch; batch::Station* station = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), Ident())); wassert(actual(station->report) == "synop"); wassert(actual(station->id) == MISSING_INT); wassert(actual(station->coords) == Coords(11.0, 45.0)); wassert_true(station->ident.is_missing()); wassert_true(station->is_new); wassert_true(station->station_data.ids_by_code.empty()); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); batch::Station* station1 = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), Ident())); wassert(actual(station) == station1); station = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), "AB123")); wassert(actual(station->report) == "synop"); wassert(actual(station->id) == MISSING_INT); wassert(actual(station->coords) == Coords(11.0, 45.0)); wassert_false(station->ident.is_missing()); wassert(actual(station->ident) == "AB123"); wassert_true(station->is_new); wassert_true(station->station_data.ids_by_code.empty()); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); station1 = wcallchecked(batch.get_station(trc, "synop", Coords(11.0, 45.0), "AB123")); wassert(actual(station) == station1); }); add_method("from_db", [](Fixture& f) { using namespace dballe::db::v7; db::v7::Tracer<> trc; Coords coords(44.5008, 11.3288); TestDataSet ds; ds.stations["synop"].station.coords = coords; ds.stations["synop"].station.report = "synop"; ds.stations["synop"].values.set("B07030", 78); // Height ds.data["synop"].station = ds.stations["synop"].station; ds.data["synop"].datetime = Datetime(2013, 10, 16, 10); ds.data["synop"].level = Level(1, 0, 0); ds.data["synop"].trange = Trange::instant(); ds.data["synop"].values.set(WR_VAR(0, 12, 101), 16.5); wassert(f.populate(ds)); Batch& batch = f.tr->batch; batch.clear(); batch::Station* station = wcallchecked(batch.get_station(trc, "synop", coords, Ident())); wassert(actual(station->report) == "synop"); wassert(actual(station->id) != MISSING_INT); wassert(actual(station->coords) == coords); wassert_true(station->ident.is_missing()); wassert_false(station->is_new); wassert_false(station->station_data.loaded); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); wassert_true(station->station_data.ids_by_code.empty()); station->get_station_data(trc); wassert_true(station->station_data.loaded); wassert_true(station->station_data.to_insert.empty()); wassert_true(station->station_data.to_update.empty()); wassert(actual(station->station_data.ids_by_code.size()) == 1u); { auto it = station->station_data.ids_by_code.begin(); wassert(actual(it->varcode) == WR_VAR(0, 7, 30)); wassert(actual(it->id) > 0u); } auto measured_data = station->get_measured_data(trc, Datetime(2013, 10, 16, 10)); wassert(actual(measured_data.datetime) == Datetime(2013, 10, 16, 10)); wassert_true(measured_data.to_insert.empty()); wassert_true(measured_data.to_update.empty()); wassert(actual(measured_data.ids_on_db.size()) == 1u); { auto it = measured_data.ids_on_db.begin(); wassert(actual(it->id_varcode.id) > 0); wassert(actual(it->id_varcode.varcode) == WR_VAR(0, 12, 101)); wassert(actual(it->id) > 0u); } }); add_method("import", [](Fixture& f) { db::v7::Tracer<> trc; impl::Messages msgs1 = read_msgs("bufr/test-airep1.bufr", Encoding::BUFR); auto opts = DBImportOptions::create(); opts->import_attributes = true; opts->update_station = true; f.tr->import_message(*msgs1[0], *opts); db::v7::Batch& batch = f.tr->batch; wassert(actual(batch.count_select_stations) == 1u); wassert(actual(batch.count_select_station_data) == 0u); wassert(actual(batch.count_select_data) == 0u); }); add_method("insert", [](Fixture& f) { using namespace db::v7; db::v7::Tracer<> trc; Batch& batch = f.tr->batch; batch.set_write_attrs(false); auto st = batch.get_station(trc, "synop", Coords(45.0, 11.0), Ident()); auto& st_data = st->get_station_data(trc); auto& data = st->get_measured_data(trc, Datetime(2018, 6, 1)); Var sv(var(WR_VAR(0, 7, 30), 1000.0)); st_data.add(&sv, batch::ERROR); Var dv(var(WR_VAR(0, 12, 101), 25.6)); int id_levtr = f.tr->levtr().obtain_id(trc, LevTrEntry(Level(1), Trange(254))); data.add(id_levtr, &dv, batch::ERROR); batch.write_pending(trc); wassert(actual(batch.count_select_stations) == 1u); wassert(actual(batch.count_select_station_data) == 0u); wassert(actual(batch.count_select_data) == 0u); }); add_method("insert_double_station_value", [](Fixture& f) { using namespace db::v7; db::v7::Tracer<> trc; Batch& batch = f.tr->batch; batch.set_write_attrs(false); auto st = batch.get_station(trc, "synop", Coords(45.0, 11.0), Ident()); auto& st_data = st->get_station_data(trc); Var sv1(var(WR_VAR(0, 7, 30), 1000.0)); Var sv2(var(WR_VAR(0, 7, 30), 1001.0)); st_data.add(&sv1, batch::ERROR); st_data.add(&sv2, batch::ERROR); batch.write_pending(trc); wassert(actual(batch.count_select_stations) == 1u); wassert(actual(batch.count_select_station_data) == 0u); wassert(actual(batch.count_select_data) == 0u); // Query var and check that it is 1001 auto cur = f.tr->query_station_data(core::Query()); wassert(actual(cur->remaining()) == 1); wassert(cur->next()); wassert(actual(cur->get_var()) == sv2); }); add_method("insert_double_measured_value", [](Fixture& f) { using namespace db::v7; db::v7::Tracer<> trc; Batch& batch = f.tr->batch; batch.set_write_attrs(false); auto st = batch.get_station(trc, "synop", Coords(45.0, 11.0), Ident()); auto& data = st->get_measured_data(trc, Datetime(2018, 6, 1)); Var dv1(var(WR_VAR(0, 12, 101), 25.6)); Var dv2(var(WR_VAR(0, 12, 101), 25.7)); int id_levtr = f.tr->levtr().obtain_id(trc, LevTrEntry(Level(1), Trange(254))); data.add(id_levtr, &dv1, batch::ERROR); data.add(id_levtr, &dv2, batch::ERROR); batch.write_pending(trc); wassert(actual(batch.count_select_stations) == 1u); wassert(actual(batch.count_select_station_data) == 0u); wassert(actual(batch.count_select_data) == 0u); // Query var and check that it is 25.7 auto cur = f.tr->query_data(core::Query()); wassert(actual(cur->remaining()) == 1); wassert(cur->next()); wassert(actual(cur->get_var()) == dv2); }); } } dballe-8.6/dballe/db/v7/qbuilder.cc0000644000175000017500000006075113554564112014034 00000000000000#include "qbuilder.h" #include "transaction.h" #include "dballe/core/defs.h" #include "dballe/core/aliases.h" #include "dballe/core/query.h" #include "dballe/core/varmatch.h" #include "dballe/var.h" #include "dballe/db/v7/repinfo.h" #include #include #include #include #include "config.h" #ifdef HAVE_LIBPQ #include "dballe/sql/postgresql.h" #endif #ifdef HAVE_MYSQL #include "dballe/sql/mysql.h" #endif using namespace std; using namespace wreport; using dballe::sql::Querybuf; using dballe::sql::ServerType; namespace dballe { namespace db { namespace v7 { static Varcode parse_varcode(const char* str, regmatch_t pos) { Varcode res; /* Parse the varcode */ if (str[pos.rm_so] == 'B') res = WR_STRING_TO_VAR(str + pos.rm_so + 1); else res = varcode_alias_resolve_substring(str + pos.rm_so, pos.rm_eo - pos.rm_so); if (res == 0) error_consistency::throwf("cannot resolve the variable code or alias in \"%.*s\"", pos.rm_eo - pos.rm_so, str + pos.rm_so); return res; } static void parse_value(const char* str, regmatch_t pos, Varinfo info, char* value) { /* Parse the value */ const char* s = str + pos.rm_so; int len = pos.rm_eo - pos.rm_so; switch (info->type) { case Vartype::String: { // Copy the string, escaping quotes int i = 0, j = 0; value[j++] = '\''; for (; i < len && j < 253; ++i, ++j) { if (s[i] == '\'') value[j++] = '\\'; value[j] = s[i]; } value[j++] = '\''; value[j] = 0; break; } case Vartype::Binary: throw error_consistency("cannot use a *_filter on a binary variable"); case Vartype::Integer: case Vartype::Decimal: { double dval; if (sscanf(s, "%lf", &dval) != 1) error_consistency::throwf("value in \"%.*s\" must be a number", len, s); sprintf(value, "%d", info->encode_decimal(dval)); break; } } } static Varinfo decode_data_filter(const std::string& filter, const char** op, const char** val, const char** val1) { static regex_t* re_normal = NULL; static regex_t* re_between = NULL; regmatch_t matches[4]; static char oper[5]; static char value[255]; static char value1[255]; #if 0 size_t len = strcspn(filter, "<=>"); const char* s = filter + len; #endif Varcode code; /* Compile the regular expression if it has not yet been done */ if (re_normal == NULL) { re_normal = new regex_t; if (int res = regcomp(re_normal, "^([^<=>]+)([<=>]+)([^<=>]+)$", REG_EXTENDED)) throw error_regexp(res, re_normal, "compiling regular expression to match normal filters"); } if (re_between == NULL) { re_between = new regex_t; if (int res = regcomp(re_between, "^([^<=>]+)<=([^<=>]+)<=([^<=>]+)$", REG_EXTENDED)) throw error_regexp(res, re_between, "compiling regular expression to match 'between' filters"); } int res = regexec(re_normal, filter.c_str(), 4, matches, 0); if (res != 0 && res != REG_NOMATCH) error_regexp::throwf(res, re_normal, "Trying to parse '%s' as a 'normal' filter", filter.c_str()); if (res == 0) { int len; /* We have a normal filter */ /* Parse the varcode */ code = parse_varcode(filter.c_str(), matches[1]); /* Query informations for the varcode */ Varinfo info = varinfo(code); /* Parse the operator */ len = matches[2].rm_eo - matches[2].rm_so; if (len > 4) error_consistency::throwf("operator %.*s is not valid", len, filter.c_str() + matches[2].rm_so); memcpy(oper, filter.c_str() + matches[2].rm_so, len); oper[len] = 0; if (strcmp(oper, "!=") == 0) *op = "<>"; else if (strcmp(oper, "==") == 0) *op = "="; else *op = oper; /* Parse the value */ parse_value(filter.c_str(), matches[3], info, value); *val = value; *val1 = NULL; return info; } else { res = regexec(re_between, filter.c_str(), 4, matches, 0); if (res == REG_NOMATCH) error_consistency::throwf("%s is not a valid filter", filter.c_str()); if (res != 0) error_regexp::throwf(res, re_normal, "Trying to parse '%s' as a 'between' filter", filter.c_str()); /* We have a between filter */ /* Parse the varcode */ code = parse_varcode(filter.c_str(), matches[2]); /* Query informations for the varcode */ Varinfo info = varinfo(code); /* No need to parse the operator */ oper[0] = 0; *op = oper; /* Parse the values */ parse_value(filter.c_str(), matches[1], info, value); parse_value(filter.c_str(), matches[3], info, value1); *val = value; *val1 = value1; return info; } } struct Constraints { const core::Query& query; const char* tbl; Querybuf& q; bool found; Constraints(const core::Query& query, const char* tbl, Querybuf& q) : query(query), tbl(tbl), q(q), found(false) {} void add_lat() { if (query.latrange.is_missing()) return; if (query.latrange.imin == query.latrange.imax) q.append_listf("%s.lat=%d", tbl, query.latrange.imin); else { if (query.latrange.imin != LatRange::IMIN) q.append_listf("%s.lat>=%d", tbl, query.latrange.imin); if (query.latrange.imax != LatRange::IMAX) q.append_listf("%s.lat<=%d", tbl, query.latrange.imax); } found = true; } void add_lon() { if (query.lonrange.is_missing()) return; if (query.lonrange.imin == query.lonrange.imax) q.append_listf("%s.lon=%d", tbl, query.lonrange.imin); else if (query.lonrange.imin < query.lonrange.imax) q.append_listf("%s.lon>=%d AND %s.lon<=%d", tbl, query.lonrange.imin, tbl, query.lonrange.imax); else q.append_listf("((%s.lon>=%d AND %s.lon<=18000000) OR (%s.lon>=-18000000 AND %s.lon<=%d))", tbl, query.lonrange.imin, tbl, tbl, tbl, query.lonrange.imax); found = true; } void add_mobile() { if (query.mobile != MISSING_INT) { if (query.mobile == 0) { q.append_listf("%s.ident IS NULL", tbl); TRACE("found fixed/mobile: adding AND %s.ident IS NULL.\n", tbl); } else { q.append_listf("NOT (%s.ident IS NULL)", tbl); TRACE("found fixed/mobile: adding AND NOT (%s.ident IS NULL)\n", tbl); } found = true; } } }; QueryBuilder::QueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars) : conn(*tr->db->conn), tr(tr), query(query), sql_query(2048), sql_from(1024), sql_where(1024), modifiers(modifiers), query_station_vars(query_station_vars) { } DataQueryBuilder::DataQueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars) : QueryBuilder(tr, query, modifiers, query_station_vars), query_attrs(modifiers & DBA_DB_MODIFIER_WITH_ATTRIBUTES) { } DataQueryBuilder::~DataQueryBuilder() { delete attr_filter; } void QueryBuilder::build() { build_select(); sql_where.start_list(" AND "); bool has_where = build_where(); if (query.limit != MISSING_INT && conn.server_type == ServerType::ORACLE) { sql_where.append_listf("rownum <= %d", query.limit); has_where = true; } // Finalise the query sql_query.append(sql_from); if (has_where) { sql_query.append(" WHERE "); sql_query.append(sql_where); } // Append ORDER BY as needed if (!(modifiers & DBA_DB_MODIFIER_UNSORTED)) { if (query.limit != MISSING_INT && conn.server_type == ServerType::ORACLE) throw error_unimplemented("sorted queries with result limit are not implemented for Oracle"); build_order_by(); } // Append LIMIT if requested if (query.limit != MISSING_INT && conn.server_type != ServerType::ORACLE) sql_query.appendf(" LIMIT %d", query.limit); } void StationQueryBuilder::build_select() { sql_query.append("SELECT s.id, s.rep, s.lat, s.lon, s.ident"); sql_from.append( " FROM station s" ); select_station = true; } bool StationQueryBuilder::build_where() { bool has_where = false; // Add pseudoana-specific where parts has_where |= add_pa_where("s"); /* * Querying var= or varlist= on a station query means querying stations * that measure that variable or those variables. */ switch (query.varcodes.size()) { case 0: break; case 1: sql_where.append_listf("EXISTS(SELECT id FROM data s_stvar" " WHERE s_stvar.id_station=s.id" " AND s_stvar.code=%d)", *query.varcodes.begin()); has_where = true; break; default: sql_where.append_listf("EXISTS(SELECT id FROM data s_stvar" " WHERE s_stvar.id_station=s.id" " AND s_stvar.code IN ("); sql_where.append_varlist(query.varcodes); sql_where.append("))"); has_where = true; break; } if (!query.report.empty()) { int src_val = tr->repinfo().get_id(query.report.c_str()); if (src_val == -1) { sql_where.append_listf("1=0"); TRACE("rep_memo %s not found: adding AND 1=0\n", query.report.c_str()); } else { sql_where.append_listf("s.rep=%d", src_val); TRACE("found rep_memo %s: adding AND s.rep=%d\n", query.report.c_str(), src_val); } has_where = true; } return has_where; } void StationQueryBuilder::build_order_by() { // https://github.com/ARPA-SIMC/dballe/issues/18 // confirmed that nothing relies on the ordering of stations in station // queries //sql_query.append(" ORDER BY s.id"); } void DataQueryBuilder::build_select() { if (query_station_vars) sql_query.append("SELECT s.id, s.rep, s.lat, s.lon, s.ident, d.code, d.id, d.value"); else sql_query.append("SELECT s.id, s.rep, s.lat, s.lon, s.ident, d.id_levtr, d.code, d.id, d.datetime, d.value"); if (query_attrs || !query.attr_filter.empty()) { sql_query.append(", d.attrs"); select_attrs = true; if (!query.attr_filter.empty()) { delete attr_filter; attr_filter = Varmatch::parse(query.attr_filter).release(); } } select_station = true; select_varinfo = true; select_data_id = true; select_data = true; sql_from.append(" FROM station s"); if (query_station_vars) { sql_from.append(" JOIN station_data d ON s.id=d.id_station"); } else { sql_from.append(" JOIN data d ON s.id=d.id_station"); sql_from.append(" JOIN levtr ltr ON ltr.id=d.id_levtr"); } } bool DataQueryBuilder::build_where() { bool has_where = false; // Add pseudoana-specific where parts has_where = add_pa_where("s") || has_where; if (!query_station_vars) { has_where = add_dt_where("d") || has_where; has_where = add_ltr_where("ltr") || has_where; } has_where = add_varcode_where("d") || has_where; has_where = add_repinfo_where("s") || has_where; has_where = add_datafilter_where("d") || has_where; //has_where = add_attrfilter_where("d") || has_where; return has_where; } bool DataQueryBuilder::match_attrs(const Var& var) const { for (const Var* a = var.next_attr(); a != NULL; a = a->next_attr()) if ((*attr_filter)(*a)) return true; return false; } #if 0 bool DataQueryBuilder::add_attrfilter_where(const char* tbl) { if (query.attr_filter.empty()) return false; const char *op, *value, *value1; Varinfo info = decode_data_filter(query.attr_filter, &op, &value, &value1); const char* atbl = query_station_vars ? "station_attr" : "attr"; sql_from.appendf(" JOIN %s %s_atf ON %s.id=%s_atf.id_data AND %s_atf.code=%d", atbl, tbl, tbl, tbl, tbl, info->code); if (value[0] == '\'') if (value1 == NULL) sql_where.append_listf("%s_atf.value%s%s", tbl, op, value); else sql_where.append_listf("%s_atf.value BETWEEN %s AND %s", tbl, value, value1); else { const char* type = (conn.server_type == ServerType::MYSQL) ? "SIGNED" : "INT"; if (value1 == NULL) sql_where.append_listf("CAST(%s_atf.value AS %s)%s%s", tbl, type, op, value); else sql_where.append_listf("CAST(%s_atf.value AS %s) BETWEEN %s AND %s", tbl, type, value, value1); } return true; } #endif void DataQueryBuilder::build_order_by() { if (modifiers & DBA_DB_MODIFIER_BEST) sql_query.append(" ORDER BY s.lat, s.lon, s.ident"); else sql_query.append(" ORDER BY d.id_station"); if (!query_station_vars) { sql_query.append(", d.datetime"); sql_query.append(", ltr.ltype1, ltr.l1, ltr.ltype2, ltr.l2, ltr.pind, ltr.p1, ltr.p2"); } if (modifiers & DBA_DB_MODIFIER_BEST) sql_query.append(", s.rep"); sql_query.append(", d.code"); } void IdQueryBuilder::build_select() { sql_query.append("SELECT d.id"); if (!query.attr_filter.empty()) { sql_query.append(", d.attrs"); select_attrs = true; } select_data_id = true; sql_from.append(" FROM station s"); if (query_station_vars) sql_from.append(" JOIN station_data d ON s.id = d.id_station"); else { sql_from.append(" JOIN data d ON s.id = d.id_station"); sql_from.append(" JOIN levtr ltr ON ltr.id = d.id_levtr"); } } void IdQueryBuilder::build_order_by() { // No ordering required } void SummaryQueryBuilder::build_select() { if (!query.attr_filter.empty()) throw error_consistency("attr_filter is not supported on summary queries"); if (modifiers & DBA_DB_MODIFIER_SUMMARY_DETAILS) { if (query_station_vars) sql_query.append("SELECT s.id, s.rep, s.lat, s.lon, s.ident, d.code, COUNT(1)"); else sql_query.append(R"( SELECT s.id, s.rep, s.lat, s.lon, s.ident, d.id_levtr, d.code, COUNT(1), MIN(d.datetime), MAX(d.datetime) )"); select_summary_details = true; } else { if (query_station_vars) sql_query.append("SELECT DISTINCT s.id, s.rep, s.lat, s.lon, s.ident, d.code"); else sql_query.append("SELECT DISTINCT s.id, s.rep, s.lat, s.lon, s.ident, d.id_levtr, d.code"); } select_station = true; select_varinfo = true; /* // Abuse id_data and datetime for count and min(datetime) stm.bind_out(output_seq++, cur.sqlrec.out_id_data); stm.bind_out(output_seq++, cur.sqlrec.out_datetime); stm.bind_out(output_seq++, cur_s.out_datetime_max); */ sql_from.append(" FROM station s"); if (query_station_vars) sql_from.append(" JOIN station_data d ON s.id = d.id_station"); else { sql_from.append(" JOIN data d ON s.id = d.id_station"); sql_from.append(" JOIN levtr ltr ON ltr.id=d.id_levtr"); } } void SummaryQueryBuilder::build_order_by() { // No ordering required, but we may add a GROUP BY if (modifiers & DBA_DB_MODIFIER_SUMMARY_DETAILS) { if (query_station_vars) sql_query.append(" GROUP BY s.id, d.code"); else sql_query.append(" GROUP BY s.id, d.id_levtr, d.code"); } } bool QueryBuilder::add_pa_where(const char* tbl) { Constraints c(query, tbl, sql_where); if (query.ana_id != MISSING_INT) { sql_where.append_listf("%s.id=%d", tbl, query.ana_id); c.found = true; } c.add_lat(); c.add_lon(); c.add_mobile(); if (!query.ident.is_missing()) { if (false) { // This is only here to move the other optional bits into else ifs // that can be compiled out ; #if HAVE_LIBPQ } else if (dynamic_cast(&conn)) { sql_where.append_listf("%s.ident=$1::text", tbl); bind_in_ident = query.ident.get(); TRACE("found ident: adding AND %s.ident=$1::text. val is %s\n", tbl, query.ident.get()); #endif #if HAVE_MYSQL } else if (dballe::sql::MySQLConnection* c = dynamic_cast(&conn)) { string escaped = c->escape(query.ident.get()); sql_where.append_listf("%s.ident='%s'", tbl, escaped.c_str()); TRACE("found ident: adding AND %s.ident='%s'. val is %s\n", tbl, escaped.c_str(), query.ident.get()); #endif } else { sql_where.append_listf("%s.ident=?", tbl); bind_in_ident = query.ident.get(); TRACE("found ident: adding AND %s.ident = ?. val is %s\n", tbl, query.ident.get()); } c.found = true; } if (query.block != MISSING_INT) { // No need to escape since the variable is integer sql_where.append_listf("EXISTS(SELECT id FROM station_data %s_blo WHERE %s_blo.id_station=%s.id" " AND %s_blo.code=257 AND %s_blo.value='%d')", tbl, tbl, tbl, tbl, tbl, query.block); c.found = true; } if (query.station != MISSING_INT) { sql_where.append_listf("EXISTS(SELECT id FROM station_data %s_sta WHERE %s_sta.id_station=%s.id" " AND %s_sta.code=258 AND %s_sta.value='%d')", tbl, tbl, tbl, tbl, tbl, query.station); c.found = true; } if (!query.ana_filter.empty()) { const char *op, *value, *value1; Varinfo info = decode_data_filter(query.ana_filter, &op, &value, &value1); sql_where.append_listf("EXISTS(SELECT id FROM station_data %s_af WHERE %s_af.id_station=%s.id" " AND %s_af.code=%d", tbl, tbl, tbl, tbl, info->code); if (value[0] == '\'') if (value1 == NULL) sql_where.appendf(" AND %s_af.value%s%s)", tbl, op, value); else sql_where.appendf(" AND %s_af.value BETWEEN %s AND %s)", tbl, value, value1); else { const char* type = (conn.server_type == ServerType::MYSQL) ? "SIGNED" : "INT"; if (value1 == NULL) sql_where.appendf(" AND CAST(%s_af.value AS %s)%s%s)", tbl, type, op, value); else sql_where.appendf(" AND CAST(%s_af.value AS %s) BETWEEN %s AND %s)", tbl, type, value, value1); } c.found = true; } return c.found; } bool QueryBuilder::add_dt_where(const char* tbl) { if (query_station_vars) return false; bool found = false; if (!query.dtrange.is_missing()) { Datetime dtmin = query.dtrange.min; Datetime dtmax = query.dtrange.max; if (dtmin == dtmax) { // Add constraint on the exact date interval sql_where.append_listf("%s.datetime=", tbl); conn.add_datetime(sql_where, dtmin); TRACE("found exact time: adding AND %s.datetime=%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu\n", tbl, dtmin.year, dtmin.month, dtmin.day, dtmin.hour, dtmin.minute, dtmin.second); found = true; } else { if (!dtmin.is_missing()) { // Add constraint on the minimum date interval sql_where.append_listf("%s.datetime>=", tbl); conn.add_datetime(sql_where, dtmin); TRACE("found min time: adding AND %s.datetime>=%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu\n", tbl, dtmin.year, dtmin.month, dtmin.day, dtmin.hour, dtmin.minute, dtmin.second); found = true; } if (!dtmax.is_missing()) { sql_where.append_listf("%s.datetime<=", tbl); conn.add_datetime(sql_where, dtmax); TRACE("found max time: adding AND %s.datetime<=%04hu-%02hhu-%02hhu%c%02hhu:%02hhu:%02hhu\n", tbl, dtmax.year, dtmax.month, dtmax.day, dtmax.hour, dtmax.minute, dtmax.second); found = true; } } } return found; } bool QueryBuilder::add_ltr_where(const char* tbl) { if (query_station_vars) return false; bool found = false; if (query.level.ltype1 != MISSING_INT) { sql_where.append_listf("%s.ltype1=%d", tbl, query.level.ltype1); found = true; } if (query.level.l1 != MISSING_INT) { sql_where.append_listf("%s.l1=%d", tbl, query.level.l1); found = true; } if (query.level.ltype2 != MISSING_INT) { sql_where.append_listf("%s.ltype2=%d", tbl, query.level.ltype2); found = true; } if (query.level.l2 != MISSING_INT) { sql_where.append_listf("%s.l2=%d", tbl, query.level.l2); found = true; } if (query.trange.pind != MISSING_INT) { sql_where.append_listf("%s.pind=%d", tbl, query.trange.pind); found = true; } if (query.trange.p1 != MISSING_INT) { sql_where.append_listf("%s.p1=%d", tbl, query.trange.p1); found = true; } if (query.trange.p2 != MISSING_INT) { sql_where.append_listf("%s.p2=%d", tbl, query.trange.p2); found = true; } return found; } bool QueryBuilder::add_varcode_where(const char* tbl) { bool found = false; switch (query.varcodes.size()) { case 0: break; case 1: sql_where.append_listf("%s.code=%d", tbl, (int)*query.varcodes.begin()); TRACE("found b: adding AND %s.code=%d\n", tbl, (int)*query.varcodes.begin()); found = true; break; default: sql_where.append_listf("%s.code IN (", tbl); sql_where.append_varlist(query.varcodes); sql_where.append(")"); TRACE("found blist: adding AND %s.code IN (...%zd items...)\n", tbl, query.varcodes.size()); found = true; break; } return found; } bool QueryBuilder::add_repinfo_where(const char* tbl) { bool found = false; if (query.priomin != MISSING_INT || query.priomax != MISSING_INT) { // Filter the repinfo cache and build a IN query std::vector ids = tr->repinfo().ids_by_prio(query); if (ids.empty()) { // No repinfo matches, so we just introduce a false value sql_where.append_list("1=0"); } else { sql_where.append_listf("%s.rep IN (", tbl); for (std::vector::const_iterator i = ids.begin(); i != ids.end(); ++i) { if (i == ids.begin()) sql_where.appendf("%d", *i); else sql_where.appendf(",%d", *i); } sql_where.append(")"); } found = true; } if (!query.report.empty()) { int src_val = tr->repinfo().get_id(query.report.c_str()); if (src_val == -1) { sql_where.append_listf("1=0"); TRACE("rep_memo %s not found: adding AND 1=0\n", query.report.c_str()); } else { sql_where.append_listf("%s.rep=%d", tbl, src_val); TRACE("found rep_memo %s: adding AND %s.rep=%d\n", query.report.c_str(), tbl, (int)src_val); } found = true; } return found; } bool QueryBuilder::add_datafilter_where(const char* tbl) { if (query.data_filter.empty()) return false; const char *op, *value, *value1; Varinfo info = decode_data_filter(query.data_filter, &op, &value, &value1); sql_where.append_listf("%s.code=%d", tbl, (int)info->code); if (value[0] == '\'') if (value1 == NULL) sql_where.append_listf("%s.value%s%s", tbl, op, value); else sql_where.append_listf("%s.value BETWEEN %s AND %s", tbl, value, value1); else { const char* type = (conn.server_type == ServerType::MYSQL) ? "SIGNED" : "INT"; if (value1 == NULL) sql_where.append_listf("CAST(%s.value AS %s)%s%s", tbl, type, op, value); else sql_where.append_listf("CAST(%s.value AS %s) BETWEEN %s AND %s", tbl, type, value, value1); } return true; } } } } dballe-8.6/dballe/db/v7/repinfo.cc0000644000175000017500000002354313554564112013665 00000000000000#include "repinfo.h" #include "dballe/db/db.h" #include "dballe/core/query.h" #include "dballe/core/csv.h" #include #include using namespace wreport; using namespace std; namespace dballe { namespace db { namespace v7 { Repinfo::Repinfo(dballe::sql::Connection& conn) : conn(conn) { } const char* Repinfo::get_rep_memo(int id) { if (const repinfo::Cache* c = get_by_id(id)) return c->memo.c_str(); error_notfound::throwf("rep_memo not found for report code %d", id); } int Repinfo::get_id(const char* memo) { char lc_memo[20]; int i; for (i = 0; i < 19 && memo[i]; ++i) lc_memo[i] = tolower(memo[i]); lc_memo[i] = 0; if (memo_idx.empty()) rebuild_memo_idx(); int pos = cache_find_by_memo(lc_memo); if (pos == -1) return -1; return memo_idx[pos].id; } int Repinfo::get_priority(const std::string& report) { const repinfo::Cache* ri_entry = get_by_memo(report.c_str()); return ri_entry ? ri_entry->prio : INT_MAX; } std::map Repinfo::get_priorities() { std::map res; for (std::vector::const_iterator i = cache.begin(); i != cache.end(); ++i) res[i->memo] = i->prio; return res; } int Repinfo::obtain_id(const char* memo) { char lc_memo[20]; int i; for (i = 0; i < 19 && memo[i]; ++i) lc_memo[i] = tolower(memo[i]); lc_memo[i] = 0; if (memo_idx.empty()) rebuild_memo_idx(); int pos = cache_find_by_memo(lc_memo); if (pos == -1) { insert_auto_entry(lc_memo); read_cache(); return get_id(lc_memo); } return memo_idx[pos].id; } std::vector Repinfo::ids_by_prio(const core::Query& q) { vector res; for (std::vector::const_iterator i = cache.begin(); i != cache.end(); ++i) { if (q.priomin != MISSING_INT && i->prio < q.priomin) continue; if (q.priomax != MISSING_INT && i->prio > q.priomax) continue; res.push_back(i->id); } return res; } const repinfo::Cache* Repinfo::get_by_id(unsigned id) const { int pos = cache_find_by_id(id); return pos == -1 ? NULL : &(cache[pos]); } const repinfo::Cache* Repinfo::get_by_memo(const char* memo) const { int pos = cache_find_by_memo(memo); if (pos == -1) return NULL; return get_by_id(memo_idx[pos].id); } int Repinfo::cache_find_by_id(unsigned id) const { /* Binary search the ID */ int begin, end; begin = -1, end = cache.size(); while (end - begin > 1) { int cur = (end + begin) / 2; if (cache[cur].id > id) end = cur; else begin = cur; } if (begin == -1 || cache[begin].id != id) return -1; else return begin; } int Repinfo::cache_find_by_memo(const char* memo) const { /* Binary search the memo index */ int begin, end; begin = -1, end = cache.size(); while (end - begin > 1) { int cur = (end + begin) / 2; if (memo_idx[cur].memo > memo) end = cur; else begin = cur; } if (begin == -1 || memo_idx[begin].memo != memo) return -1; else return begin; } void Repinfo::cache_append(unsigned id, const char* memo, const char* desc, int prio, const char* descriptor, int tablea) { /* Ensure that we are adding things in order */ if (!cache.empty() && cache.back().id >= id) error_consistency::throwf( "checking that value to append to repinfo cache (%u) " "is greather than the last value in che cache (%u)", id, (unsigned)cache.back().id); memo_idx.clear(); /* Enlarge buffer if needed */ cache.push_back(repinfo::Cache(id, memo, desc, prio, descriptor, tablea)); } void Repinfo::rebuild_memo_idx() const { memo_idx.clear(); memo_idx.resize(cache.size()); for (size_t i = 0; i < cache.size(); ++i) { memo_idx[i].memo = cache[i].memo; memo_idx[i].id = cache[i].id; } std::sort(memo_idx.begin(), memo_idx.end()); } namespace { struct fd_closer { FILE* fd; fd_closer(FILE* fd) : fd(fd) {} ~fd_closer() { fclose(fd); } }; inline void inplace_tolower(std::string& buf) { for (string::iterator i = buf.begin(); i != buf.end(); ++i) *i = tolower(*i); } } std::vector Repinfo::read_repinfo_file(const char* deffile) { if (deffile == 0) deffile = DB::default_repinfo_file(); /* Open the input CSV file */ FILE* in = fopen(deffile, "r"); if (in == NULL) error_system::throwf("opening file %s", deffile); fd_closer closer(in); /* Read the CSV file */ vector newitems; vector columns; for (int line = 1; csv_read_next(in, columns); ++line) { int id, pos; if (columns.size() != 6) error_parse::throwf(deffile, line, "Expected 6 columns, got %zd", columns.size()); // Lowercase all rep_memos inplace_tolower(columns[1]); id = strtol(columns[0].c_str(), 0, 10); pos = cache_find_by_id(id); if (pos == -1) { /* New entry */ newitems.push_back(repinfo::Cache(id, columns[1], columns[2], strtol(columns[3].c_str(), 0, 10), columns[4], strtol(columns[5].c_str(), 0, 10))); newitems.back().make_new(); } else { /* Possible update on an existing entry */ cache[pos].new_memo = columns[1]; cache[pos].new_desc = columns[2]; cache[pos].new_prio = strtol(columns[3].c_str(), 0, 10); cache[pos].new_descriptor = columns[4]; cache[pos].new_tablea = strtol(columns[5].c_str(), 0, 10); } } /* Verify conflicts */ for (size_t i = 0; i < cache.size(); ++i) { /* Skip empty items or items that will be deleted */ if (cache[i].memo.empty() || cache[i].new_memo.empty()) continue; if (cache[i].memo != cache[i].new_memo) error_consistency::throwf("cannot rename rep_cod %d (previous rep_memo was %s, new rep_memo is %s)", (int)cache[i].id, cache[i].memo.c_str(), cache[i].new_memo.c_str()); for (size_t j = i + 1; j < cache.size(); ++j) { /* Skip empty items or items that will be deleted */ if (cache[j].memo.empty() || cache[j].new_memo.empty()) continue; if (cache[j].new_prio == cache[i].new_prio) error_consistency::throwf("%s has the same priority (%d) as %s", cache[j].new_memo.c_str(), (int)cache[j].new_prio, cache[i].new_memo.c_str()); } for (vector::const_iterator j = newitems.begin(); j != newitems.end(); ++j) { if (j->new_prio == cache[i].new_prio) error_consistency::throwf("%s has the same priority (%d) as %s", j->new_memo.c_str(), (int)j->new_prio, cache[i].new_memo.c_str()); } } for (vector::const_iterator i = newitems.begin(); i != newitems.end(); ++i) { /*fprintf(stderr, "prio %d\n", cur->item.new_prio);*/ for (vector::const_iterator j = i + 1; j != newitems.end(); ++j) { if (i->new_prio == j->new_prio) error_consistency::throwf("%s has the same priority (%d) as %s", i->new_memo.c_str(), (int)i->new_prio, j->new_memo.c_str()); } } /* fprintf(stderr, "POST PARSE:\n"); for (const auto& e: cache) e.dump(stderr); */ return newitems; } void Repinfo::update(const char* deffile, int* added, int* deleted, int* updated) { *added = *deleted = *updated = 0; // Read the new repinfo data from file vector newitems = read_repinfo_file(deffile); // Verify that we are not trying to delete a repinfo entry that is // in use for (const auto& entry : cache) { /* Ensure that we are not deleting a repinfo entry that is already in use */ if (!entry.memo.empty() && entry.new_memo.empty()) if (id_use_count(entry.id, entry.memo.c_str()) > 0) error_consistency::throwf( "trying to delete repinfo entry %u,%s which is currently in use", (unsigned)entry.id, entry.memo.c_str()); } // Perform the changes for (const auto& entry : cache) { if (!entry.memo.empty() && entry.new_memo.empty()) { // Delete the items that were deleted */ delete_entry(entry.id); ++*deleted; } else if (!entry.memo.empty() && !entry.new_memo.empty()) { // Update the items that were modified update_entry(entry); ++*updated; } } // Insert the new items for (const auto& entry : newitems) { insert_entry(entry); ++*added; } // Reread the cache read_cache(); } namespace repinfo { Cache::Cache(int id, const std::string& memo, const std::string& desc, int prio, const std::string& descriptor, int tablea) : id(id), memo(memo), desc(desc), prio(prio), descriptor(descriptor), tablea(tablea), new_prio(0), new_tablea(0) { } void Cache::make_new() { new_memo = memo; new_desc = desc; new_prio = prio; new_descriptor = descriptor; new_tablea = tablea; } void Cache::dump(FILE* out) const { fprintf(stderr, "%u: %s %s %d %s %u (%s %s %d %s %u)\n", id, memo.c_str(), desc.c_str(), prio, descriptor.c_str(), tablea, new_memo.c_str(), new_desc.c_str(), new_prio, new_descriptor.c_str(), new_tablea); } bool Memoidx::operator<(const Memoidx& val) const { return memo < val.memo; } } } } } dballe-8.6/dballe/db/v7/data.cc0000644000175000017500000000701213554564112013125 00000000000000#include "data.h" #include "dballe/types.h" #include "dballe/values.h" #include #include using namespace std; using namespace wreport; namespace dballe { namespace db { namespace v7 { const char* StationDataTraits::table_name = "station_data"; const char* DataTraits::table_name = "data"; template const char* DataCommon::table_name = Traits::table_name; template void DataCommon::read_attrs_into_values(Tracer<>& trc, int id_data, Values& values) { read_attrs(trc, id_data, [&](unique_ptr var) { values.set(move(var)); }); } template void DataCommon::read_attrs_into_values(Tracer<>& trc, int id_data, Values& values, const db::AttrList& exclude) { read_attrs(trc, id_data, [&](unique_ptr var) { if (std::find(exclude.begin(), exclude.end(), var->code()) == exclude.end()) values.set(move(var)); }); } template void DataCommon::merge_attrs(Tracer<>& trc, int id_data, const Values& attrs) { // Read existing attributes Values merged; read_attrs_into_values(trc, id_data, merged); // Merge attributes from attrs merged.merge(attrs); // Write them back write_attrs(trc, id_data, merged); } template void DataCommon::remove_attrs(Tracer<>& trc, int id_data, const db::AttrList& attrs) { if (attrs.empty()) remove_all_attrs(trc, id_data); else { // Read existing attributes Values remaining; read_attrs_into_values(trc, id_data, remaining, attrs); if (remaining.empty()) remove_all_attrs(trc, id_data); else write_attrs(trc, id_data, remaining); } } template class DataCommon; template class DataCommon; StationDataDumper::StationDataDumper(FILE* out) : out(out) { } void StationDataDumper::print_head() { fprintf(out, "dump of table station_data:\n"); fprintf(out, " id st var\n"); } void StationDataDumper::print_row(int id, int id_station, wreport::Varcode code, const char* val, const std::vector& attrs) { fprintf(out, " %4d %4d %01d%02d%03d", id, id_station, WR_VAR_FXY(code)); if (!val) fprintf(out, "\n"); else fprintf(out, " %s\n", val); DBValues::decode(attrs, [&](std::unique_ptr var) { fprintf(out, " "); var->print(out); }); ++count; } void StationDataDumper::print_tail() { fprintf(out, "%d element%s in table data\n", count, count != 1 ? "s" : ""); } DataDumper::DataDumper(FILE* out) : out(out) { } void DataDumper::print_head() { fprintf(out, "dump of table data:\n"); fprintf(out, " id st ltr datetime var\n"); } void DataDumper::print_row(int id, int id_station, int id_levtr, const Datetime& dt, wreport::Varcode code, const char* val, const std::vector& attrs) { fprintf(out, " %4d %4d %04d %04d-%02d-%02d %02d:%02d:%02d %01d%02d%03d", id, id_station, id_levtr, dt.year, dt.month, dt.day, dt.hour, dt.minute, dt.second, WR_VAR_FXY(code)); if (!val) fprintf(out, "\n"); else fprintf(out, " %s\n", val); DBValues::decode(attrs, [&](std::unique_ptr var) { fprintf(out, " "); var->print(out); }); ++count; } void DataDumper::print_tail() { fprintf(out, "%d element%s in table data\n", count, count != 1 ? "s" : ""); } } } } dballe-8.6/dballe/db/v7/import.cc0000644000175000017500000000741013554573614013537 00000000000000#include "db.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/batch.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/station.h" #include "dballe/db/v7/levtr.h" #include "dballe/db/v7/data.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include using namespace wreport; using dballe::sql::Connection; namespace dballe { namespace db { namespace v7 { void Transaction::add_msg_to_batch(Tracer<>& trc, const Message& message, const dballe::DBImportOptions& opts) { const impl::Message& msg = impl::Message::downcast(message); batch::Station* station; // Coordinates Coords coords = msg.get_coords(); if (coords.is_missing()) throw error_notfound("coordinates not found in data to import"); // Report code std::string report; if (!opts.report.empty()) report = opts.report; else report = msg.get_report(); // Station identifier Ident ident = msg.get_ident(); station = batch.get_station(trc, report, coords, ident); if (opts.update_station || (station->is_new && station->id == MISSING_INT)) { for (const auto& var: msg.station_data) { Varcode code = var->code(); // Do not import datetime in the station info context, unless it has attributes if (code == WR_VAR(0, 4, 1) && !var->next_attr()) continue; if (code == WR_VAR(0, 4, 2) && !var->next_attr()) continue; if (code == WR_VAR(0, 4, 3) && !var->next_attr()) continue; if (code == WR_VAR(0, 4, 4) && !var->next_attr()) continue; if (code == WR_VAR(0, 4, 5) && !var->next_attr()) continue; if (code == WR_VAR(0, 4, 6) && !var->next_attr()) continue; station->get_station_data(trc).add(var.get(), opts.overwrite ? batch::UPDATE : batch::IGNORE); } } // Fill the bulk insert with the rest of the data v7::LevTr& lt = levtr(); // Defer creation of MeasuredData to prevent complaining about missing // datetime info if we have no data to import batch::MeasuredData* md = nullptr; for (const auto& ctx: msg.data) { int id_levtr = -1; for (const auto& val: ctx.values) { if (not val->isset()) continue; if (!opts.varlist.empty() && std::find(opts.varlist.begin(), opts.varlist.end(), val->code()) == opts.varlist.end()) continue; if (!md) { Datetime datetime = msg.get_datetime(); if (datetime.is_missing()) throw error_notfound("date/time informations not found (or incomplete) in message to insert"); md = &station->get_measured_data(trc, datetime); } if (id_levtr == -1) { // Get the database ID of the lev_tr id_levtr = lt.obtain_id(trc, LevTrEntry(ctx.level, ctx.trange)); } md->add(id_levtr, val.get(), opts.overwrite ? batch::UPDATE : batch::IGNORE); } } } void Transaction::import_message(const dballe::Message& message, const dballe::DBImportOptions& opts) { Tracer<> trc(this->trc ? this->trc->trace_import(1) : nullptr); batch.set_write_attrs(opts.import_attributes); add_msg_to_batch(trc, message, opts); // Run the bulk insert batch.write_pending(trc); } void Transaction::import_messages(const std::vector>& messages, const dballe::DBImportOptions& opts) { Tracer<> trc(this->trc ? this->trc->trace_import(messages.size()) : nullptr); batch.set_write_attrs(opts.import_attributes); for (const auto& i: messages) add_msg_to_batch(trc, *i, opts); // Run the bulk insert batch.write_pending(trc); } } } } dballe-8.6/dballe/db/v7/data.h0000644000175000017500000001273013554573614013001 00000000000000#ifndef DBALLE_DB_V7_DATAV7_H #define DBALLE_DB_V7_DATAV7_H #include #include #include #include #include #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { template class DataCommon { protected: typedef typename Traits::BatchValue BatchValue; static const char* table_name; v7::Transaction& tr; /** * Load attributes from the database into a Values */ void read_attrs_into_values(Tracer<>& trc, int id_data, Values& values); /** * Load attributes from the database into a Values, except those whose * Varcode is in `exclude` */ void read_attrs_into_values(Tracer<>& trc, int id_data, Values& values, const db::AttrList& exclude); /** * Replace the attributes of a variable with those in Values */ virtual void write_attrs(Tracer<>& trc, int id_data, const Values& values) = 0; /** * Remove all attributes from a variable */ virtual void remove_all_attrs(Tracer<>& trc, int id_data) = 0; public: DataCommon(v7::Transaction& tr) : tr(tr) {} virtual ~DataCommon() {} /** * Load from the database all the attributes for var * * @param trc * Operation tracer using for debugging and diagnostics * @param id_data * ID of the data row for the value of which we will read attributes * @param dest * Function that will be called to consume the attrbutes as they are * loaded. */ virtual void read_attrs(Tracer<>& trc, int id_data, std::function)> dest) = 0; /** * Merge the given attributes with the existing attributes of the given * variable: * * * Existing attributes not in attrs are preserved. * * Existing attributes in attrs are overwritten. * * New attributes in attrs are inesrted. */ void merge_attrs(Tracer<>& trc, int id_data, const Values& attrs); /** * Remove the given attributes from the given variable, if they exist. */ void remove_attrs(Tracer<>& trc, int data_id, const db::AttrList& attrs); /// Bulk variable update virtual void update(Tracer<>& trc, std::vector& vars, bool with_attrs) = 0; /// Run the query to delete all records selected by the given QueryBuilder virtual void remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) = 0; /// Run the query to delete the record with the given ID virtual void remove_by_id(Tracer<>& trc, int id) = 0; /// Dump the entire contents of the table to an output stream virtual void dump(FILE* out) = 0; virtual void clear_cache() = 0; }; struct StationDataDumper { unsigned count = 0; FILE* out; StationDataDumper(FILE* out); void print_head(); void print_row(int id, int id_station, wreport::Varcode code, const char* val, const std::vector& attrs); void print_tail(); }; struct DataDumper { unsigned count = 0; FILE* out; DataDumper(FILE* out); void print_head(); void print_row(int id, int id_station, int id_levtr, const Datetime& dt, wreport::Varcode code, const char* val, const std::vector& attrs); void print_tail(); }; struct StationDataTraits { typedef batch::StationDatum BatchValue; static const char* table_name; }; struct DataTraits { typedef batch::MeasuredDatum BatchValue; static const char* table_name; }; extern template class DataCommon; extern template class DataCommon; struct StationData : public DataCommon { using DataCommon::DataCommon; /// Bulk variable insert virtual void insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) = 0; /// Query contents of the data table virtual void query(Tracer<>& trc, int id_station, std::function dest) = 0; /** * Run a station data query, iterating on the resulting variables */ virtual void run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) = 0; }; struct Data : public DataCommon { using DataCommon::DataCommon; /// Bulk variable insert virtual void insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) = 0; /// Query contents of the data table virtual void query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) = 0; /** * Run a data query, iterating on the resulting variables */ virtual void run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) = 0; /** * Run a summary query, iterating on the resulting variables */ virtual void run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function) = 0; }; } } } #endif dballe-8.6/dballe/db/v7/transaction.h0000644000175000017500000000743413554564112014413 00000000000000#ifndef DBALLE_DB_V7_TRANSACTION_H #define DBALLE_DB_V7_TRANSACTION_H #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { struct Transaction : public dballe::db::Transaction { protected: /// Report information v7::Repinfo* m_repinfo = nullptr; /// Station information v7::Station* m_station = nullptr; /// Level/timerange information v7::LevTr* m_levtr = nullptr; /// Station data v7::StationData* m_station_data = nullptr; /// Variable data v7::Data* m_data = nullptr; void add_msg_to_batch(Tracer<>& trc, const Message& message, const dballe::DBImportOptions& opts); public: typedef v7::DB DB; std::shared_ptr db; /// SQL-side transaction dballe::sql::Transaction* sql_transaction = nullptr; /// True if commit or rollback have already been called on this transaction bool fired = false; /// Batch importer v7::Batch batch; /// Tracing system v7::Tracer trc; Transaction(std::shared_ptr db, std::unique_ptr sql_transaction); Transaction(const Transaction&) = delete; Transaction(Transaction&&) = delete; Transaction& operator=(const Transaction&) = delete; Transaction& operator=(Transaction&&) = delete; ~Transaction(); /// Access the repinfo table v7::Repinfo& repinfo(); /// Access the station table v7::Station& station(); /// Access the levtr table v7::LevTr& levtr(); /// Access the station_data table v7::StationData& station_data(); /// Access the data table v7::Data& data(); void commit() override; void rollback() override; void rollback_nothrow() noexcept override; void clear_cached_state() override; std::unique_ptr query_stations(const Query& query); std::unique_ptr query_station_data(const Query& query) override; std::unique_ptr query_data(const Query& query); std::unique_ptr query_summary(const Query& query); std::unique_ptr query_messages(const Query& query); void attr_query_station(int data_id, std::function)> dest) override; void attr_query_data(int data_id, std::function)> dest) override; void insert_station_data(dballe::Data& vals, const dballe::DBInsertOptions& opts=dballe::DBInsertOptions::defaults) override; void insert_data(dballe::Data& vals, const dballe::DBInsertOptions& opts=dballe::DBInsertOptions::defaults) override; void remove_station_data(const Query& query) override; void remove_data(const Query& query) override; void remove_station_data_by_id(int id); void remove_data_by_id(int id); void remove_all() override; void attr_insert_station(int data_id, const Values& attrs) override; void attr_insert_data(int data_id, const Values& attrs) override; void attr_remove_station(int data_id, const db::AttrList& attrs) override; void attr_remove_data(int data_id, const db::AttrList& attrs) override; void import_message(const Message& message, const dballe::DBImportOptions& opts) override; void import_messages(const std::vector>& msgs, const dballe::DBImportOptions& opts) override; void update_repinfo(const char* repinfo_file, int* added, int* deleted, int* updated) override; static Transaction& downcast(dballe::db::Transaction& transaction); void dump(FILE* out) override; }; struct TestTransaction : public Transaction { using Transaction::Transaction; void commit() override; }; } } } #endif dballe-8.6/dballe/db/v7/levtr.cc0000644000175000017500000000232413554564112013351 00000000000000#include "levtr.h" #include "dballe/msg/msg.h" using namespace std; namespace dballe { namespace db { namespace v7 { LevTr::LevTr(v7::Transaction& tr) : tr(tr) {} LevTr::~LevTr() {} void LevTr::clear_cache() { cache.clear(); } const LevTrEntry& LevTr::lookup_cache(int id) { const LevTrEntry* res = cache.find_entry(id); if (!res) wreport::error_notfound::throwf("LevTr with ID %d not found in cache", id); return *res; } impl::msg::Context* LevTr::to_msg(Tracer<>& trc, int id, impl::Message& msg) { auto i = lookup_id(trc, id); impl::msg::Context& res = msg.obtain_context(i->level, i->trange); return &res; } void LevTr::dump(FILE* out) { int count = 0; fprintf(out, "dump of table levtr:\n"); fprintf(out, " id lev tr\n"); _dump([&](int id, const Level& level, const Trange& trange) { fprintf(out, " %4d ", id); int written = level.print(out, "-", ""); while (written++ < 21) putc(' ', out); written = trange.print(out, "-", ""); while (written++ < 11) putc(' ', out); putc('\n', out); ++count; }); fprintf(out, "%d element%s in table levtr\n", count, count != 1 ? "s" : ""); } } } } dballe-8.6/dballe/db/v7/fwd.h0000644000175000017500000000251113554564112012635 00000000000000#ifndef DBALLE_DB_V7_FWD_H #define DBALLE_DB_V7_FWD_H namespace dballe { namespace db { namespace v7 { struct Transaction; struct QueryBuilder; struct StationQueryBuilder; struct DataQueryBuilder; struct SummaryQueryBuilder; struct IdQueryBuilder; struct DB; struct Repinfo; struct Station; struct LevTr; struct LevTrEntry; struct SQLTrace; struct Driver; namespace cursor { struct Stations; struct StationData; struct Data; struct Summary; } namespace batch { struct Station; struct StationDatum; struct MeasuredDatum; } namespace trace { struct Step; struct Transaction; } /** * Smart pointer for trace::Step objects, which calls done() when going out of * scope */ template class Tracer { protected: Step* step; public: Tracer() : step(nullptr) {} Tracer(Step* step) : step(step) {} Tracer(const Tracer&) = delete; Tracer(Tracer&& o) : step(o.step) { o.step = nullptr; } Tracer& operator=(const Tracer&) = delete; Tracer& operator=(Tracer&&) = delete; ~Tracer() { if (step) step->done(); } void reset(Step* step) { this->step = step; } void done() { if (step) step->done(); step = nullptr; } Step* operator->() { return step; } operator bool() const { return step; } }; } } } #endif dballe-8.6/dballe/db/v7/utils.h0000644000175000017500000000230613554564112013217 00000000000000#ifndef DBALLE_DB_V7_UTILS_H #define DBALLE_DB_V7_UTILS_H #include #include namespace dballe { namespace db { namespace v7 { struct IdVarcode { int id; wreport::Varcode varcode; IdVarcode(int id, wreport::Varcode varcode) : id(id), varcode(varcode) { } bool operator==(const IdVarcode& o) const { return std::tie(id, varcode) == std::tie(o.id, o.varcode); } bool operator!=(const IdVarcode& o) const { return std::tie(id, varcode) != std::tie(o.id, o.varcode); } bool operator<(const IdVarcode& o) const { return std::tie(id, varcode) < std::tie(o.id, o.varcode); } bool operator>(const IdVarcode& o) const { return std::tie(id, varcode) > std::tie(o.id, o.varcode); } }; } } } namespace std { template<> struct hash { typedef dballe::db::v7::IdVarcode argument_type; typedef std::size_t result_type; result_type operator()(argument_type const& s) const noexcept { result_type const h1 ( std::hash{}(s.id) ); result_type const h2 ( std::hash{}(s.varcode) ); return h1 ^ (h2 << 1); } }; } #endif dballe-8.6/dballe/db/v7/trace-test.cc0000644000175000017500000000042413554564112014267 00000000000000#include "dballe/db/tests.h" using namespace std; using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } test("db_trace"); void Tests::register_tests() { } } dballe-8.6/dballe/db/v7/repinfo-test.cc0000644000175000017500000001056713554564112014644 00000000000000#include "dballe/db/tests.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/repinfo.h" #include #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace std; using namespace wreport; namespace { template class Tests : public FixtureTestCase> { using FixtureTestCase>::FixtureTestCase; typedef EmptyTransactionFixture Fixture; void register_tests() override { // Test simple queries this->add_method("query", [](Fixture& f) { auto& ri = f.tr->repinfo(); wassert(actual(ri.get_id("synop")) == 1); wassert(actual(ri.get_id("generic")) == 255); wassert(actual(ri.get_rep_memo(1)) == "synop"); wassert(actual(ri.get_priority("synop")) == 101); wassert(actual(ri.get_priority("wrong")) == INT_MAX); }); // Test update this->add_method("update", [](Fixture& f) { auto& ri = f.tr->repinfo(); wassert(actual(ri.get_id("synop")) == 1); int added, deleted, updated; ri.update(NULL, &added, &deleted, &updated); wassert(actual(added) == 0); wassert(actual(deleted) == 0); wassert(actual(updated) == 13); wassert(actual(ri.get_id("synop")) == 1); }); // Test update from a file that was known to fail this->add_method("fail", [](Fixture& f) { auto& ri = f.tr->repinfo(); wassert(actual(ri.get_id("synop")) == 1); int added, deleted, updated; ri.update((string(getenv("DBA_TESTDATA")) + "/test-repinfo1.csv").c_str(), &added, &deleted, &updated); wassert(actual(added) == 3); wassert(actual(deleted) == 11); wassert(actual(updated) == 2); wassert(actual(ri.get_id("synop")) == 1); wassert(actual(ri.get_id("FIXspnpo")) == 201); }); // Test update from a file with a negative priority this->add_method("fail1", [](Fixture& f) { auto& ri = f.tr->repinfo(); wassert(actual(ri.get_priority("generic")) == 1000); int added, deleted, updated; wassert(ri.update((string(getenv("DBA_TESTDATA")) + "/test-repinfo2.csv").c_str(), &added, &deleted, &updated)); wassert(actual(added) == 3); wassert(actual(deleted) == 11); wassert(actual(updated) == 2); wassert(actual(ri.get_priority("generic")) == -5); }); // Test automatic repinfo creation this->add_method("fail2", [](Fixture& f) { auto& ri = f.tr->repinfo(); int id = ri.obtain_id("foobar"); wassert(actual(id) > 0); wassert(actual(ri.get_rep_memo(id)) == "foobar"); wassert(actual(ri.get_priority("foobar")) == 1001); id = ri.obtain_id("barbaz"); wassert(actual(id) > 0); wassert(actual(ri.get_rep_memo(id)) == "barbaz"); wassert(actual(ri.get_priority("barbaz")) == 1002); }); // See https://github.com/ARPA-SIMC/dballe/issues/30 this->add_method("case", [](Fixture& f) { auto& ri = f.tr->repinfo(); int added, deleted, updated; sys::write_file("test_case.csv", "234,foOBar,foOBar,100,oss,0\n"); ri.update("test_case.csv", &added, &deleted, &updated); wassert(actual(added) == 1); wassert(actual(deleted) == 13); wassert(actual(updated) == 0); int id = ri.obtain_id("fooBAR"); wassert(actual(id) == 234); wassert(actual(ri.get_rep_memo(id)) == "foobar"); wassert(actual(ri.get_priority("foobar")) == 100); wassert(actual(ri.get_id("foobar")) == id); wassert(actual(ri.get_id("Foobar")) == id); wassert(actual(ri.get_id("FOOBAR")) == id); wassert(actual(ri.obtain_id("FOOBAR")) == id); }); } }; Tests test_sqlitev7("db_v7_repinfo_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests test_psqlv7("db_v7_repinfo_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests test_mysqlv7("db_v7_repinfo_mysql", "MYSQL"); #endif } dballe-8.6/dballe/db/v7/station.cc0000644000175000017500000000137313554564112013701 00000000000000#include "station.h" #include "dballe/core/values.h" #include "transaction.h" using namespace wreport; using namespace dballe::db; using namespace std; namespace dballe { namespace db { namespace v7 { Station::Station(v7::Transaction& tr) : tr(tr) { } Station::~Station() { } void Station::dump(FILE* out) { int count = 0; fprintf(out, "dump of table station:\n"); _dump([&](int id, int rep, const Coords& coords, const char* ident) { fprintf(out, " %d, %d, %.5f, %.5f", id, rep, coords.dlat(), coords.dlon()); if (!ident) putc('\n', out); else fprintf(out, ", %s\n", ident); ++count; }); fprintf(out, "%d element%s in table station\n", count, count != 1 ? "s" : ""); } } } } dballe-8.6/dballe/db/v7/export.cc0000644000175000017500000001264113572414716013545 00000000000000#include "db.h" #include "cursor.h" #include "qbuilder.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/driver.h" #include "dballe/db/v7/station.h" #include "dballe/db/v7/levtr.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include "dballe/core/query.h" #include #include #include #include using namespace wreport; using namespace std; using dballe::sql::Connection; namespace dballe { namespace db { namespace v7 { namespace { struct StationValues : public Values { Tracer<>& trc; StationValues(Tracer<>& trc) : trc(trc) {} void read(v7::Transaction& tr, int id_station) { clear(); tr.station().get_station_vars(trc, id_station, [&](std::unique_ptr var) { set(std::move(var)); }); } }; struct ProtoVar { int id_levtr; std::unique_ptr var; ProtoVar(int id_levtr, std::unique_ptr var) : id_levtr(id_levtr), var(std::move(var)) {} }; struct ProtoMessage { std::unique_ptr msg; std::vector vars; ProtoMessage() : msg(new impl::Message) {} }; struct Cursor : public impl::CursorMessage { typedef std::vector> Results; Results results; Results::iterator cur; bool at_start = true; bool has_value() const { return !at_start && cur != results.end(); } const Message& get_message() const override { return **cur; } std::unique_ptr detach_message() override { return std::move(*cur); } int remaining() const override { if (at_start) return results.size(); return results.end() - cur; } bool next() override { if (at_start) { cur = results.begin(); at_start = false; } else if (cur != results.end()) ++cur; return cur != results.end(); } void discard() override { cur = results.end(); } DBStation get_station() const override { DBStation res; res.coords = (*cur)->get_coords(); res.ident = (*cur)->get_ident(); res.report = (*cur)->get_report(); return res; } }; } std::unique_ptr Transaction::query_messages(const Query& query) { Tracer<> trc(this->trc ? this->trc->trace_export_msgs(query) : nullptr); v7::LevTr& lt = levtr(); // The big export query DataQueryBuilder qb(dynamic_pointer_cast(shared_from_this()), core::Query::downcast(query), DBA_DB_MODIFIER_SORT_FOR_EXPORT | DBA_DB_MODIFIER_WITH_ATTRIBUTES, false); qb.build(); // Current context information used to detect context changes Datetime last_datetime; int last_ana_id = -1; StationValues station_values(trc); if (db->explain_queries) { fprintf(stderr, "EXPLAIN "); query.print(stderr); db->conn->explain(qb.sql_query, stderr); } // Retrieve results, buffering them locally to avoid performing concurrent // queries std::map> results; std::set id_levtrs; ProtoMessage* msg; data().run_data_query(trc, qb, [&](const dballe::DBStation& station, int id_levtr, const Datetime& datetime, int id_data, std::unique_ptr var) { if (station.id != last_ana_id || datetime != last_datetime) { auto& vec = results[station.id]; vec.emplace_back(); msg = &vec.back(); msg->msg->set_datetime(datetime); msg->msg->station_data.set(newvar(WR_VAR(0, 1, 194), station.report)); msg->msg->type = impl::Message::type_from_repmemo(station.report.c_str()); msg->msg->station_data.set(newvar(WR_VAR(0, 5, 1), station.coords.lat)); msg->msg->station_data.set(newvar(WR_VAR(0, 6, 1), station.coords.lon)); if (!station.ident.is_missing()) msg->msg->station_data.set(newvar(WR_VAR(0, 1, 11), (const char*)station.ident)); last_datetime = datetime; last_ana_id = station.id; } id_levtrs.insert(id_levtr); msg->vars.emplace_back(id_levtr, std::move(var)); }); lt.prefetch_ids(trc, id_levtrs); std::unique_ptr res(new Cursor); for (auto& r: results) { station_values.read(*this, r.first); for (auto& msg: r.second) { // Fill in station information msg.msg->station_data.merge(station_values); // Move variables to contexts int last_id_levtr = -1; impl::msg::Context* ctx = nullptr; for (auto& pvar: msg.vars) { if (pvar.id_levtr != last_id_levtr) { ctx = lt.to_msg(trc, pvar.id_levtr, *msg.msg); last_id_levtr = pvar.id_levtr; } ctx->values.set(std::move(pvar.var)); } msg.vars.clear(); // Send message to consumer if (msg.msg->type == MessageType::PILOT || msg.msg->type == MessageType::TEMP || msg.msg->type == MessageType::TEMP_SHIP) msg.msg->sounding_pack_levels(); res->results.emplace_back(std::move(msg.msg)); } r.second.clear(); } results.clear(); return std::unique_ptr(res.release()); } } } } dballe-8.6/dballe/db/v7/utils-test.cc0000644000175000017500000000135313554564112014333 00000000000000#include "dballe/db/tests.h" #include "dballe/db/v7/utils.h" #include "config.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { struct Fixture : EmptyTransactionFixture { using EmptyTransactionFixture::EmptyTransactionFixture; }; class Tests : public FixtureTestCase { using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg1("db_v7_utils_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg3("db_v7_utils_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg4("db_v7_utils_mysql", "MYSQL"); #endif void Tests::register_tests() { add_method("empty", [](Fixture& f) { using namespace dballe::db::v7; }); } } dballe-8.6/dballe/db/v7/sqlite/0000755000175000017500000000000013602152021013250 500000000000000dballe-8.6/dballe/db/v7/sqlite/levtr.h0000644000175000017500000000255613554564112014523 00000000000000#ifndef DBALLE_DB_V7_SQLITE_LEVTRV7_H #define DBALLE_DB_V7_SQLITE_LEVTRV7_H #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace sqlite { struct DB; /** * Precompiled queries to manipulate the lev_tr table */ struct SQLiteLevTr : public v7::LevTr { protected: /** * DB connection. */ dballe::sql::SQLiteConnection& conn; /** Precompiled select statement */ dballe::sql::SQLiteStatement* sstm = nullptr; /** Precompiled select data statement */ dballe::sql::SQLiteStatement* sdstm = nullptr; /** Precompiled insert statement */ dballe::sql::SQLiteStatement* istm = nullptr; /** Precompiled delete statement */ dballe::sql::SQLiteStatement* dstm = nullptr; void _dump(std::function out) override; public: SQLiteLevTr(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn); SQLiteLevTr(const LevTr&) = delete; SQLiteLevTr(const LevTr&&) = delete; SQLiteLevTr& operator=(const SQLiteLevTr&) = delete; ~SQLiteLevTr(); void prefetch_ids(Tracer<>& trc, const std::set& id) override; const LevTrEntry* lookup_id(Tracer<>& trc, int id) override; int obtain_id(Tracer<>& trc, const LevTrEntry& desc) override; }; } } } } #endif dballe-8.6/dballe/db/v7/sqlite/station.h0000644000175000017500000000337613554564112015051 00000000000000#ifndef DBALLE_DB_V7_SQLITE_STATION_H #define DBALLE_DB_V7_SQLITE_STATION_H #include #include #include namespace wreport { struct Var; } namespace dballe { namespace db { namespace v7 { namespace sqlite { /** * Precompiled queries to manipulate the station table */ class SQLiteStation : public v7::Station { protected: /** * DB connection. */ dballe::sql::SQLiteConnection& conn; /** Precompiled select fixed station query */ dballe::sql::SQLiteStatement* sfstm = nullptr; /** Precompiled select mobile station query */ dballe::sql::SQLiteStatement* smstm = nullptr; /** Precompiled insert query */ dballe::sql::SQLiteStatement* istm = nullptr; /** Precompiled select station data query */ dballe::sql::SQLiteStatement* ssdstm = nullptr; void _dump(std::function out) override; public: SQLiteStation(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn); ~SQLiteStation(); SQLiteStation(const SQLiteStation&) = delete; SQLiteStation(const SQLiteStation&&) = delete; SQLiteStation& operator=(const SQLiteStation&) = delete; DBStation lookup(Tracer<>& trc, int id_station) override; int maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) override; int insert_new(Tracer<>& trc, const dballe::DBStation& desc) override; void get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) override; void add_station_vars(Tracer<>& trc, int id_station, DBValues& values) override; void run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function) override; }; } } } } #endif dballe-8.6/dballe/db/v7/sqlite/driver.h0000644000175000017500000000155513554564112014660 00000000000000#ifndef DBALLE_DB_V7_SQLITE_DRIVER_H #define DBALLE_DB_V7_SQLITE_DRIVER_H #include #include namespace dballe { namespace db { namespace v7 { namespace sqlite { struct Driver : public v7::Driver { dballe::sql::SQLiteConnection& conn; Driver(dballe::sql::SQLiteConnection& conn); virtual ~Driver(); std::unique_ptr create_repinfo(v7::Transaction& tr) override; std::unique_ptr create_station(v7::Transaction& tr) override; std::unique_ptr create_levtr(v7::Transaction& tr) override; std::unique_ptr create_station_data(v7::Transaction& tr) override; std::unique_ptr create_data(v7::Transaction& tr) override; void create_tables_v7() override; void delete_tables_v7() override; void vacuum_v7() override; }; } } } } #endif dballe-8.6/dballe/db/v7/sqlite/repinfo.h0000644000175000017500000000231613554564112015023 00000000000000#ifndef DBALLE_DB_V7_SQLITE_REPINFO_H #define DBALLE_DB_V7_SQLITE_REPINFO_H #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace sqlite { /** * Fast cached access to the repinfo table */ struct SQLiteRepinfoV7 : public v7::Repinfo { /** * DB connection. The pointer is assumed always valid during the * lifetime of the object */ dballe::sql::SQLiteConnection& conn; SQLiteRepinfoV7(dballe::sql::SQLiteConnection& conn); SQLiteRepinfoV7(const SQLiteRepinfoV7&) = delete; SQLiteRepinfoV7(const SQLiteRepinfoV7&&) = delete; virtual ~SQLiteRepinfoV7(); SQLiteRepinfoV7& operator=(const SQLiteRepinfoV7&) = delete; void dump(FILE* out) override; protected: /// Return how many time this ID is used in the database int id_use_count(unsigned id, const char* name) override; void delete_entry(unsigned id) override; void update_entry(const v7::repinfo::Cache& entry) override; void insert_entry(const v7::repinfo::Cache& entry) override; void read_cache() override; void insert_auto_entry(const char* memo) override; }; } } } } #endif dballe-8.6/dballe/db/v7/sqlite/repinfo.cc0000644000175000017500000000733413554564112015166 00000000000000#include "repinfo.h" #include "dballe/db/db.h" #include "dballe/sql/sqlite.h" using namespace wreport; using namespace std; using dballe::sql::SQLiteConnection; namespace dballe { namespace db { namespace v7 { namespace sqlite { SQLiteRepinfoV7::SQLiteRepinfoV7(SQLiteConnection& conn) : Repinfo(conn), conn(conn) { read_cache(); } SQLiteRepinfoV7::~SQLiteRepinfoV7() { } void SQLiteRepinfoV7::read_cache() { cache.clear(); memo_idx.clear(); auto stm = conn.sqlitestatement("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); stm->execute([&]() { string memo = stm->column_string(1); string desc = stm->column_string(2); string descriptor = stm->column_string(4); cache_append( stm->column_int(0), memo.c_str(), desc.c_str(), stm->column_int(3), descriptor.c_str(), stm->column_int(5) ); }); // Rebuild the memo index as well rebuild_memo_idx(); } void SQLiteRepinfoV7::insert_auto_entry(const char* memo) { auto stm = conn.sqlitestatement("SELECT MAX(id) FROM repinfo"); unsigned id; stm->execute_one([&]() { id = stm->column_int(0); }); stm = conn.sqlitestatement("SELECT MAX(prio) FROM repinfo"); unsigned prio; stm->execute_one([&]() { prio = stm->column_int(0); }); ++id; ++prio; stm = conn.sqlitestatement(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES (?, ?, ?, ?, '-', 255) )"); stm->bind(id, memo, memo, prio); stm->execute(); } int SQLiteRepinfoV7::id_use_count(unsigned id, const char* name) { unsigned count = 0; auto stm = conn.sqlitestatement("SELECT COUNT(1) FROM station WHERE rep=?"); stm->bind(id); stm->execute_one([&]() { count = stm->column_int(0); }); return count; } void SQLiteRepinfoV7::delete_entry(unsigned id) { auto stm = conn.sqlitestatement("DELETE FROM repinfo WHERE id=?"); stm->bind(id); stm->execute(); } void SQLiteRepinfoV7::update_entry(const v7::repinfo::Cache& entry) { auto stm = conn.sqlitestatement(R"( UPDATE repinfo set memo=?, description=?, prio=?, descriptor=?, tablea=? WHERE id=? )"); stm->bind( entry.new_memo, entry.new_desc, entry.new_prio, entry.new_descriptor.c_str(), entry.new_tablea, entry.id); stm->execute(); } void SQLiteRepinfoV7::insert_entry(const v7::repinfo::Cache& entry) { auto stm = conn.sqlitestatement(R"( INSERT INTO repinfo (id, memo, description, prio, descriptor, tablea) VALUES (?, ?, ?, ?, ?, ?) )"); stm->bind( entry.id, entry.new_memo, entry.new_desc, entry.new_prio, entry.new_descriptor, entry.new_tablea); stm->execute(); } void SQLiteRepinfoV7::dump(FILE* out) { fprintf(out, "dump of table repinfo:\n"); fprintf(out, " id memo description prio desc tablea\n"); int count = 0; auto stm = conn.sqlitestatement("SELECT id, memo, description, prio, descriptor, tablea FROM repinfo ORDER BY id"); stm->execute([&]() { string memo = stm->column_string(1); string desc = stm->column_string(2); string descriptor = stm->column_string(4); fprintf(out, " %4d %s %s %d %s %d\n", stm->column_int(0), memo.c_str(), desc.c_str(), stm->column_int(3), descriptor.c_str(), stm->column_int(5)); ++count; }); fprintf(out, "%d element%s in table repinfo\n", count, count != 1 ? "s" : ""); } } } } } dballe-8.6/dballe/db/v7/sqlite/data.cc0000644000175000017500000003407613554564112014440 00000000000000#include "data.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/batch.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/trace.h" #include "dballe/sql/sqlite.h" #include "dballe/sql/querybuf.h" #include "dballe/values.h" #include "dballe/core/values.h" #include "dballe/core/varmatch.h" #include #include using namespace wreport; using namespace std; using dballe::sql::SQLiteConnection; using dballe::sql::SQLiteStatement; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace sqlite { template class SQLiteDataCommon; template class SQLiteDataCommon; template SQLiteDataCommon::SQLiteDataCommon(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn) : Parent(tr), conn(conn) { char query[64]; snprintf(query, 64, "UPDATE %s set value=?, attrs=? WHERE id=?", Parent::table_name); ustm = conn.sqlitestatement(query).release(); } template SQLiteDataCommon::~SQLiteDataCommon() { delete read_attrs_stm; delete write_attrs_stm; delete remove_attrs_stm; delete sstm; delete istm; delete ustm; } template void SQLiteDataCommon::read_attrs(Tracer<>& trc, int id_data, std::function)> dest) { if (!read_attrs_stm) { char query[64]; snprintf(query, 64, "SELECT attrs FROM %s WHERE id=?", Parent::table_name); read_attrs_stm = conn.sqlitestatement(query).release(); } Tracer<> trc_sel(trc ? trc->trace_select("SELECT attrs FROM … WHERE id=?") : nullptr); read_attrs_stm->bind_val(1, id_data); read_attrs_stm->execute_one([&]() { if (trc_sel) trc_sel->add_row(); Values::decode(read_attrs_stm->column_blob(0), dest); }); } template void SQLiteDataCommon::write_attrs(Tracer<>& trc, int id_data, const Values& values) { if (!write_attrs_stm) { char query[64]; snprintf(query, 64, "UPDATE %s SET attrs=? WHERE id=?", Parent::table_name); write_attrs_stm = conn.sqlitestatement(query).release(); } Tracer<> trc_upd(trc ? trc->trace_update("UPDATE … SET attrs=? WHERE id=?", 1) : nullptr); vector encoded = values.encode(); write_attrs_stm->bind_val(1, encoded); write_attrs_stm->bind_val(2, id_data); write_attrs_stm->execute(); } template void SQLiteDataCommon::remove_all_attrs(Tracer<>& trc, int id_data) { if (!remove_attrs_stm) { char query[64]; snprintf(query, 64, "UPDATE %s SET attrs=NULL WHERE id=?", Parent::table_name); remove_attrs_stm = conn.sqlitestatement(query).release(); } Tracer<> trc_upd(trc ? trc->trace_update("UPDATE … SET attrs=NULL WHERE id=?", 1) : nullptr); remove_attrs_stm->bind_val(1, id_data); remove_attrs_stm->execute(); } namespace { bool match_attrs(const Varmatch& match, const std::vector& attrs) { bool found = false; Values::decode(attrs, [&](std::unique_ptr var) { if (match(*var)) found = true; }); return found; } } template void SQLiteDataCommon::remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) { char query[64]; snprintf(query, 64, "DELETE FROM %s WHERE id=?", Parent::table_name); auto stmd = conn.sqlitestatement(query); auto stm = conn.sqlitestatement(qb.sql_query); if (qb.bind_in_ident) stm->bind_val(1, qb.bind_in_ident); std::unique_ptr attr_filter; if (!qb.query.attr_filter.empty()) attr_filter = Varmatch::parse(qb.query.attr_filter); // Iterate all the data_id results, deleting the related data and attributes Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); stm->execute([&]() { if (trc_sel) trc_sel->add_row(); if (attr_filter.get() && !match_attrs(*attr_filter, stm->column_blob(1))) return; // Compile the DELETE query for the data Tracer<> trc_del(trc ? trc->trace_delete(query, 1) : nullptr); stmd->bind_val(1, stm->column_int(0)); stmd->execute(); }); } template void SQLiteDataCommon::remove_by_id(Tracer<>& trc, int id) { char query[64]; snprintf(query, 64, "DELETE FROM %s WHERE id=%d", Parent::table_name, id); // Iterate all the data_id results, deleting the related data and attributes Tracer<> trc_sel(trc ? trc->trace_delete(query, 1) : nullptr); conn.execute(query); } template void SQLiteDataCommon::update(Tracer<>& trc, std::vector& vars, bool with_attrs) { for (auto& v: vars) { ustm->bind_val(1, v.var->enqc()); core::value::Encoder enc; if (with_attrs && v.var->next_attr()) { enc.append_attributes(*v.var); ustm->bind_val(2, enc.buf); } else ustm->bind_null_val(2); ustm->bind_val(3, v.id); Tracer<> trc_upd(trc ? trc->trace_update("UPDATE … set value=?, attrs=? WHERE id=?", 1) : nullptr); ustm->execute(); } } static const char* select_station_data_query = "SELECT id, code FROM station_data WHERE id_station=?"; static const char* insert_station_data_query = "INSERT INTO station_data (id_station, code, value, attrs) VALUES (?, ?, ?, ?)"; SQLiteStationData::SQLiteStationData(v7::Transaction& tr, SQLiteConnection& conn) : SQLiteDataCommon(tr, conn) { sstm = conn.sqlitestatement(select_station_data_query).release(); istm = conn.sqlitestatement(insert_station_data_query).release(); } void SQLiteStationData::query(Tracer<>& trc, int id_station, std::function dest) { sstm->bind_val(1, id_station); Tracer<> trc_sel(trc ? trc->trace_select(select_station_data_query) : nullptr); sstm->execute([&]() { if (trc_sel) trc_sel->add_row(); int id = sstm->column_int(0); wreport::Varcode code = sstm->column_int(1); dest(id, code); }); } void SQLiteStationData::insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); istm->bind_val(1, id_station); for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; istm->bind_val(2, v->var->code()); istm->bind_val(3, v->var->enqc()); core::value::Encoder enc; if (with_attrs && v->var->next_attr()) { enc.append_attributes(*v->var); istm->bind_val(4, enc.buf); } else istm->bind_null_val(4); Tracer<> trc_ins(trc ? trc->trace_insert(insert_station_data_query, 1) : nullptr); istm->execute(); v->id = conn.get_last_insert_id(); } } void SQLiteStationData::run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); auto stm = conn.sqlitestatement(qb.sql_query); if (qb.bind_in_ident) stm->bind_val(1, qb.bind_in_ident); dballe::DBStation station; stm->execute([&]() { if (trc_sel) trc_sel->add_row(); wreport::Varcode code = stm->column_int(5); const char* value = stm->column_string(7); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(stm->column_blob(8), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = stm->column_int(0); if (id_station != station.id) { station.id = id_station; station.report = qb.tr->repinfo().get_rep_memo(stm->column_int(1)); station.coords.lat = stm->column_int(2); station.coords.lon = stm->column_int(3); if (stm->column_isnull(4)) station.ident.clear(); else station.ident = stm->column_string(4); } int id_data = stm->column_int(6); dest(station, id_data, move(var)); }); } void SQLiteStationData::dump(FILE* out) { StationDataDumper dumper(out); dumper.print_head(); auto stm = conn.sqlitestatement("SELECT id, id_station, code, value, attrs FROM station_data"); stm->execute([&]() { const char* val = stm->column_isnull(3) ? nullptr : stm->column_string(3); dumper.print_row(stm->column_int(0), stm->column_int(1), stm->column_int(2), val, stm->column_blob(4)); }); dumper.print_tail(); } static const char* select_data_query = "SELECT id, id_levtr, code FROM data WHERE id_station=? AND datetime=?"; static const char* insert_data_query = "INSERT INTO data (id_station, id_levtr, datetime, code, value, attrs) VALUES (?, ?, ?, ?, ?, ?)"; SQLiteData::SQLiteData(v7::Transaction& tr, SQLiteConnection& conn) : SQLiteDataCommon(tr, conn) { sstm = conn.sqlitestatement(select_data_query).release(); istm = conn.sqlitestatement(insert_data_query).release(); } void SQLiteData::query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select(select_data_query) : nullptr); sstm->bind_val(1, id_station); sstm->bind_val(2, datetime); sstm->execute([&]() { if (trc_sel) trc_sel->add_row(); int id_levtr = sstm->column_int(1); wreport::Varcode code = sstm->column_int(2); int id = sstm->column_int(0); dest(id, id_levtr, code); }); } void SQLiteData::insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) { std::sort(vars.begin(), vars.end()); istm->bind_val(1, id_station); istm->bind_val(3, datetime); for (auto v = vars.begin(); v != vars.end(); ++v) { // Skip duplicates auto next = v + 1; if (next != vars.end() && *v == *next) continue; Tracer<> trc_ins(trc ? trc->trace_insert(insert_data_query, 1) : nullptr); istm->bind_val(2, v->id_levtr); istm->bind_val(4, v->var->code()); istm->bind_val(5, v->var->enqc()); core::value::Encoder enc; if (with_attrs && v->var->next_attr()) { enc.append_attributes(*v->var); istm->bind_val(6, enc.buf); } else istm->bind_null_val(6); istm->execute(); v->id = conn.get_last_insert_id(); } } void SQLiteData::run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)> dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); auto stm = conn.sqlitestatement(qb.sql_query); if (qb.bind_in_ident) stm->bind_val(1, qb.bind_in_ident); dballe::DBStation station; stm->execute([&]() { if (trc_sel) trc_sel->add_row(); wreport::Varcode code = stm->column_int(6); const char* value = stm->column_string(9); auto var = newvar(code, value); if (qb.select_attrs) core::value::Decoder::decode_attrs(stm->column_blob(10), *var); // Postprocessing filter of attr_filter if (qb.attr_filter && !qb.match_attrs(*var)) return; int id_station = stm->column_int(0); if (id_station != station.id) { station.id = id_station; station.report = tr.repinfo().get_rep_memo(stm->column_int(1)); station.coords.lat = stm->column_int(2); station.coords.lon = stm->column_int(3); if (stm->column_isnull(4)) station.ident.clear(); else station.ident = stm->column_string(4); } int id_levtr = stm->column_int(5); int id_data = stm->column_int(7); Datetime datetime = stm->column_datetime(8); dest(station, id_levtr, datetime, id_data, move(var)); }); } void SQLiteData::run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); auto stm = conn.sqlitestatement(qb.sql_query); if (qb.bind_in_ident) stm->bind_val(1, qb.bind_in_ident); dballe::DBStation station; stm->execute([&]() { if (trc_sel) trc_sel->add_row(); int id_station = stm->column_int(0); if (id_station != station.id) { station.id = id_station; station.report = qb.tr->repinfo().get_rep_memo(stm->column_int(1)); station.coords.lat = stm->column_int(2); station.coords.lon = stm->column_int(3); if (stm->column_isnull(4)) station.ident.clear(); else station.ident = stm->column_string(4); } int id_levtr = stm->column_int(5); wreport::Varcode code = stm->column_int(6); size_t count = 0; DatetimeRange datetime; if (qb.select_summary_details) { count = stm->column_int(7); datetime = DatetimeRange(stm->column_datetime(8), stm->column_datetime(9)); } dest(station, id_levtr, code, datetime, count); }); } void SQLiteData::dump(FILE* out) { DataDumper dumper(out); dumper.print_head(); auto stm = conn.sqlitestatement("SELECT id, id_station, id_levtr, datetime, code, value, attrs FROM data"); stm->execute([&]() { const char* val = stm->column_isnull(5) ? nullptr : stm->column_string(5); dumper.print_row(stm->column_int(0), stm->column_int(1), stm->column_int(2), stm->column_datetime(3), stm->column_int(4), val, stm->column_blob(6)); }); dumper.print_tail(); } } } } } dballe-8.6/dballe/db/v7/sqlite/data.h0000644000175000017500000000726113554564112014276 00000000000000#ifndef DBALLE_DB_V7_SQLITE_DATA_H #define DBALLE_DB_V7_SQLITE_DATA_H #include #include #include namespace dballe { namespace db { namespace v7 { namespace sqlite { struct DB; // Partial implementation of the common parts of StationData and Data template class SQLiteDataCommon : public Parent { protected: /// DB connection dballe::sql::SQLiteConnection& conn; /// Precompiled read attributes statement dballe::sql::SQLiteStatement* read_attrs_stm = nullptr; /// Precompiled write attributes statement dballe::sql::SQLiteStatement* write_attrs_stm = nullptr; /// Precompiled remove attributes statement dballe::sql::SQLiteStatement* remove_attrs_stm = nullptr; /// Precompiled select statement dballe::sql::SQLiteStatement* sstm = nullptr; /// Precompiled insert statement dballe::sql::SQLiteStatement* istm = nullptr; /// Precompiled update statement dballe::sql::SQLiteStatement* ustm = nullptr; public: SQLiteDataCommon(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn); SQLiteDataCommon(const SQLiteDataCommon&) = delete; SQLiteDataCommon(const SQLiteDataCommon&&) = delete; SQLiteDataCommon& operator=(const SQLiteDataCommon&) = delete; ~SQLiteDataCommon(); void update(Tracer<>& trc, std::vector& vars, bool with_attrs) override; void read_attrs(Tracer<>& trc, int id_data, std::function)> dest) override; void write_attrs(Tracer<>& trc, int id_data, const Values& values) override; void remove_all_attrs(Tracer<>& trc, int id_data) override; void remove(Tracer<>& trc, const v7::IdQueryBuilder& qb) override; void remove_by_id(Tracer<>& trc, int id) override; }; extern template class SQLiteDataCommon; extern template class SQLiteDataCommon; /** * Precompiled query to manipulate the station data table */ class SQLiteStationData : public SQLiteDataCommon { public: using SQLiteDataCommon::SQLiteDataCommon; SQLiteStationData(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn); void query(Tracer<>& trc, int id_station, std::function dest) override; void insert(Tracer<>& trc, int id_station, std::vector& vars, bool with_attrs) override; void run_station_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void dump(FILE* out) override; void clear_cache() override {} }; /** * Precompiled query to manipulate the data table */ class SQLiteData : public SQLiteDataCommon { public: using SQLiteDataCommon::SQLiteDataCommon; SQLiteData(v7::Transaction& tr, dballe::sql::SQLiteConnection& conn); void query(Tracer<>& trc, int id_station, const Datetime& datetime, std::function dest) override; void insert(Tracer<>& trc, int id_station, const Datetime& datetime, std::vector& vars, bool with_attrs) override; void run_data_query(Tracer<>& trc, const v7::DataQueryBuilder& qb, std::function var)>) override; void run_summary_query(Tracer<>& trc, const v7::SummaryQueryBuilder& qb, std::function) override; void dump(FILE* out) override; void clear_cache() override {} }; } } } } #endif dballe-8.6/dballe/db/v7/sqlite/levtr.cc0000644000175000017500000001163313554564112014655 00000000000000#include "levtr.h" #include "dballe/core/defs.h" #include "dballe/msg/msg.h" #include "dballe/sql/querybuf.h" #include "dballe/sql/sqlite.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/trace.h" #include #include #include using namespace wreport; using namespace std; using dballe::sql::SQLiteConnection; using dballe::sql::SQLiteStatement; namespace dballe { namespace db { namespace v7 { namespace sqlite { namespace { Level to_level(SQLiteStatement& stm, int first_id=0) { return Level( stm.column_int(first_id), stm.column_int(first_id + 1), stm.column_int(first_id + 2), stm.column_int(first_id + 3)); } Trange to_trange(SQLiteStatement& stm, int first_id=0) { return Trange( stm.column_int(first_id), stm.column_int(first_id + 1), stm.column_int(first_id + 2)); } } static const char* select_query = "SELECT id FROM levtr WHERE" " ltype1=? AND l1=? AND ltype2=? AND l2=?" " AND pind=? AND p1=? AND p2=?"; static const char* select_data_query = "SELECT ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id=?"; static const char* insert_query = "INSERT INTO levtr (ltype1, l1, ltype2, l2, pind, p1, p2) VALUES (?, ?, ?, ?, ?, ?, ?)"; SQLiteLevTr::SQLiteLevTr(v7::Transaction& tr, SQLiteConnection& conn) : v7::LevTr(tr), conn(conn) { // Create the statement for select fixed sstm = conn.sqlitestatement(select_query).release(); // Create the statement for select data sdstm = conn.sqlitestatement(select_data_query).release(); // Create the statement for insert istm = conn.sqlitestatement(insert_query).release(); } SQLiteLevTr::~SQLiteLevTr() { delete sstm; delete sdstm; delete istm; } void SQLiteLevTr::prefetch_ids(Tracer<>& trc, const std::set& ids) { if (ids.empty()) return; sql::Querybuf qb; if (ids.size() < 100) { qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr WHERE id IN ("); qb.start_list(","); for (auto id: ids) qb.append_listf("%d", id); qb.append(")"); } else qb.append("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr"); Tracer<> trc_sel(trc ? trc->trace_select(qb) : nullptr); auto stm = conn.sqlitestatement(qb); stm->execute([&]() { if (trc_sel) trc_sel->add_row(); cache.insert(unique_ptr(new LevTrEntry( stm->column_int(0), Level(stm->column_int(1), stm->column_int(2), stm->column_int(3), stm->column_int(4)), Trange(stm->column_int(5), stm->column_int(6), stm->column_int(7))))); }); } const LevTrEntry* SQLiteLevTr::lookup_id(Tracer<>& trc, int id) { // First look it up in the transaction cache const LevTrEntry* res = cache.find_entry(id); if (res) return res; Tracer<> trc_sel(trc ? trc->trace_select(select_data_query) : nullptr); sdstm->bind(id); sdstm->execute_one([&]() { if (trc_sel) trc_sel->add_row(); std::unique_ptr e(new LevTrEntry); e->id = id; e->level.ltype1 = sdstm->column_int(0); e->level.l1 = sdstm->column_int(1); e->level.ltype2 = sdstm->column_int(2); e->level.l2 = sdstm->column_int(3); e->trange.pind = sdstm->column_int(4); e->trange.p1 = sdstm->column_int(5); e->trange.p2 = sdstm->column_int(6); res = cache.insert(move(e)); }); if (!res) error_notfound::throwf("levtr with id %d not found in the database", id); return res; } int SQLiteLevTr::obtain_id(Tracer<>& trc, const LevTrEntry& desc) { int id = cache.find_id(desc); if (id != MISSING_INT) return id; Tracer<> trc_oid(trc ? trc->trace_select(select_query) : nullptr); sstm->bind( desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); // If there is an existing record, use its ID and don't do an INSERT sstm->execute_one([&]() { if (trc_oid) trc_oid->add_row(); id = sstm->column_int(0); }); trc_oid.done(); if (id != MISSING_INT) { cache.insert(desc, id); return id; } // Not found in the database, insert a new one trc_oid.reset(trc ? trc->trace_insert(insert_query, 1) : nullptr); istm->bind( desc.level.ltype1, desc.level.l1, desc.level.ltype2, desc.level.l2, desc.trange.pind, desc.trange.p1, desc.trange.p2); istm->execute(); id = conn.get_last_insert_id(); cache.insert(desc, id); return id; } void SQLiteLevTr::_dump(std::function out) { auto stm = conn.sqlitestatement("SELECT id, ltype1, l1, ltype2, l2, pind, p1, p2 FROM levtr ORDER BY ID"); stm->execute([&]() { out(stm->column_int(0), to_level(*stm, 1), to_trange(*stm, 5)); }); } } } } } dballe-8.6/dballe/db/v7/sqlite/station.cc0000644000175000017500000001517713554564112015211 00000000000000#include "station.h" #include "dballe/db/v7/transaction.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/repinfo.h" #include "dballe/db/v7/trace.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/sql/sqlite.h" #include "dballe/core/var.h" #include "dballe/values.h" #include using namespace wreport; using namespace dballe::db; using namespace std; using dballe::sql::SQLiteConnection; using dballe::sql::SQLiteStatement; namespace dballe { namespace db { namespace v7 { namespace sqlite { static const char* select_fixed_query = "SELECT id FROM station WHERE rep=? AND lat=? AND lon=? AND ident IS NULL"; static const char* select_mobile_query = "SELECT id FROM station WHERE rep=? AND lat=? AND lon=? AND ident=?"; static const char* insert_query = "INSERT INTO station (rep, lat, lon, ident)" " VALUES (?, ?, ?, ?);"; static const char* select_station_data_query = "SELECT rep, lat, lon, ident FROM station WHERE id=?"; SQLiteStation::SQLiteStation(v7::Transaction& tr, SQLiteConnection& conn) : v7::Station(tr), conn(conn) { // Create the statement for select fixed sfstm = conn.sqlitestatement(select_fixed_query).release(); // Create the statement for select mobile smstm = conn.sqlitestatement(select_mobile_query).release(); // Create the statement for insert istm = conn.sqlitestatement(insert_query).release(); // Create the statement for insert ssdstm = conn.sqlitestatement(select_station_data_query).release(); } SQLiteStation::~SQLiteStation() { delete sfstm; delete smstm; delete istm; delete ssdstm; } DBStation SQLiteStation::lookup(Tracer<>& trc, int id_station) { Tracer<> trc_sel; ssdstm->bind_val(1, id_station); if (trc) trc_sel.reset(trc->trace_select(select_station_data_query)); DBStation station; station.id = id_station; bool found = false; ssdstm->execute_one([&]() { if (trc_sel) trc_sel->add_row(); found = true; station.report = tr.repinfo().get_rep_memo(ssdstm->column_int(0)); station.coords.lat = ssdstm->column_int(1); station.coords.lon = ssdstm->column_int(2); if (ssdstm->column_isnull(3)) station.ident.clear(); else station.ident = ssdstm->column_string(3); }); if (found) return station; stringstream msg; msg << "Station with id " << id_station << " not found"; throw std::runtime_error(msg.str()); } int SQLiteStation::maybe_get_id(Tracer<>& trc, const dballe::DBStation& st) { SQLiteStatement* s; int rep = tr.repinfo().obtain_id(st.report.c_str()); Tracer<> trc_sel; if (st.ident.get()) { smstm->bind_val(1, rep); smstm->bind_val(2, st.coords.lat); smstm->bind_val(3, st.coords.lon); smstm->bind_val(4, st.ident.get()); s = smstm; if (trc) trc_sel.reset(trc->trace_select(select_mobile_query)); } else { sfstm->bind_val(1, rep); sfstm->bind_val(2, st.coords.lat); sfstm->bind_val(3, st.coords.lon); s = sfstm; if (trc) trc_sel.reset(trc->trace_select(select_fixed_query)); } bool found = false; int id; s->execute_one([&]() { if (trc_sel) trc_sel->add_row(); found = true; id = s->column_int(0); }); if (found) return id; else return MISSING_INT; } int SQLiteStation::insert_new(Tracer<>& trc, const dballe::DBStation& desc) { // If no station was found, insert a new one istm->bind_val(1, tr.repinfo().get_id(desc.report.c_str())); istm->bind_val(2, desc.coords.lat); istm->bind_val(3, desc.coords.lon); if (desc.ident) istm->bind_val(4, desc.ident.get()); else istm->bind_null_val(4); istm->execute(); if (trc) trc->trace_insert(insert_query, 1); return conn.get_last_insert_id(); } void SQLiteStation::get_station_vars(Tracer<>& trc, int id_station, std::function)> dest) { // Perform the query static const char query[] = R"( SELECT d.code, d.value, d.attrs FROM station_data d WHERE d.id_station=? ORDER BY d.code )"; Tracer<> trc_sel(trc ? trc->trace_select(query) : nullptr); auto stm = conn.sqlitestatement(query); stm->bind(id_station); TRACE("get_station_vars Performing query: %s with idst %d\n", query, id_station); // Retrieve results stm->execute([&]() { if (trc_sel) trc_sel->add_row(); Varcode code = stm->column_int(0); TRACE("get_station_vars Got %d%02d%03d %s\n", WR_VAR_FXY(code), stm->column_string(1)); unique_ptr var = newvar(code, stm->column_string(1)); if (!stm->column_isnull(2)) { TRACE("get_station_vars add attributes\n"); DBValues::decode(stm->column_blob(2), [&](unique_ptr a) { var->seta(move(a)); }); } dest(move(var)); }); } void SQLiteStation::add_station_vars(Tracer<>& trc, int id_station, DBValues& values) { const char* query = R"( SELECT d.code, d.value FROM station_data d WHERE d.id_station = ? )"; Tracer<> trc_sel(trc ? trc->trace_select(query) : nullptr); auto stm = conn.sqlitestatement(query); stm->bind(id_station); stm->execute([&]() { if (trc_sel) trc_sel->add_row(); values.set(newvar((wreport::Varcode)stm->column_int(0), stm->column_string(1))); }); } void SQLiteStation::run_station_query(Tracer<>& trc, const v7::StationQueryBuilder& qb, std::function dest) { Tracer<> trc_sel(trc ? trc->trace_select(qb.sql_query) : nullptr); auto stm = conn.sqlitestatement(qb.sql_query); if (qb.bind_in_ident) stm->bind_val(1, qb.bind_in_ident); dballe::DBStation station; stm->execute([&]() { if (trc_sel) trc_sel->add_row(); station.id = stm->column_int(0); station.report = tr.repinfo().get_rep_memo(stm->column_int(1)); station.coords.lat = stm->column_int(2); station.coords.lon = stm->column_int(3); if (stm->column_isnull(4)) station.ident.clear(); else station.ident = stm->column_string(4); dest(station); }); } void SQLiteStation::_dump(std::function out) { auto stm = conn.sqlitestatement("SELECT id, rep, lat, lon, ident FROM station"); stm->execute([&]() { const char* ident = stm->column_isnull(4) ? nullptr : stm->column_string(4); out(stm->column_int(0), stm->column_int(1), Coords(stm->column_int(2), stm->column_int(3)), ident); }); } } } } } dballe-8.6/dballe/db/v7/sqlite/driver.cc0000644000175000017500000001065713554564112015021 00000000000000#include "driver.h" #include "repinfo.h" #include "station.h" #include "levtr.h" #include "data.h" #include "dballe/db/v7/qbuilder.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "dballe/sql/sqlite.h" #include "dballe/var.h" #include #include using namespace std; using namespace wreport; using dballe::sql::SQLiteConnection; using dballe::sql::SQLiteStatement; using dballe::sql::Transaction; using dballe::sql::Querybuf; namespace dballe { namespace db { namespace v7 { namespace sqlite { Driver::Driver(SQLiteConnection& conn) : v7::Driver(conn), conn(conn) { } Driver::~Driver() { } std::unique_ptr Driver::create_repinfo(v7::Transaction& tr) { return unique_ptr(new SQLiteRepinfoV7(conn)); } std::unique_ptr Driver::create_station(v7::Transaction& tr) { return unique_ptr(new SQLiteStation(tr, conn)); } std::unique_ptr Driver::create_levtr(v7::Transaction& tr) { return unique_ptr(new SQLiteLevTr(tr, conn)); } std::unique_ptr Driver::create_station_data(v7::Transaction& tr) { return unique_ptr(new SQLiteStationData(tr, conn)); } std::unique_ptr Driver::create_data(v7::Transaction& tr) { return unique_ptr(new SQLiteData(tr, conn)); } void Driver::create_tables_v7() { conn.exec(R"( CREATE TABLE repinfo ( id INTEGER PRIMARY KEY, memo VARCHAR(30) NOT NULL, description VARCHAR(255) NOT NULL, prio INTEGER NOT NULL, descriptor CHAR(6) NOT NULL, tablea INTEGER NOT NULL, UNIQUE (prio), UNIQUE (memo) ); )"); conn.exec(R"( CREATE TABLE station ( id INTEGER PRIMARY KEY, rep INTEGER NOT NULL REFERENCES repinfo (id) ON DELETE CASCADE, lat INTEGER NOT NULL, lon INTEGER NOT NULL, ident CHAR(64), UNIQUE (rep, lat, lon, ident) ); CREATE INDEX pa_rep ON station(rep); CREATE INDEX pa_lon ON station(lon); )"); conn.exec(R"( CREATE TABLE levtr ( id INTEGER PRIMARY KEY, ltype1 INTEGER NOT NULL, l1 INTEGER NOT NULL, ltype2 INTEGER NOT NULL, l2 INTEGER NOT NULL, pind INTEGER NOT NULL, p1 INTEGER NOT NULL, p2 INTEGER NOT NULL, UNIQUE (ltype1, l1, ltype2, l2, pind, p1, p2) ); )"); conn.exec(R"( CREATE TABLE station_data ( id INTEGER PRIMARY KEY, id_station INTEGER NOT NULL REFERENCES station (id) ON DELETE CASCADE, code INTEGER NOT NULL, value VARCHAR(255) NOT NULL, attrs BLOB, UNIQUE (id_station, code) ); )"); conn.exec(R"( CREATE TABLE data ( id INTEGER PRIMARY KEY, id_station INTEGER NOT NULL REFERENCES station (id) ON DELETE CASCADE, id_levtr INTEGER NOT NULL REFERENCES levtr(id) ON DELETE CASCADE, datetime TEXT NOT NULL, code INTEGER NOT NULL, value VARCHAR(255) NOT NULL, attrs BLOB, UNIQUE (id_station, datetime, id_levtr, code) ); CREATE INDEX data_lt ON data(id_levtr); )"); conn.set_setting("version", "V7"); } void Driver::delete_tables_v7() { conn.drop_table_if_exists("data"); conn.drop_table_if_exists("station_data"); conn.drop_table_if_exists("levtr"); conn.drop_table_if_exists("repinfo"); conn.drop_table_if_exists("station"); conn.drop_settings(); } void Driver::vacuum_v7() { conn.exec(R"( DELETE FROM levtr WHERE id IN ( SELECT ltr.id FROM levtr ltr LEFT JOIN data d ON d.id_levtr = ltr.id WHERE d.id_levtr is NULL) )"); conn.exec(R"( DELETE FROM station_data WHERE id IN ( SELECT sd.id FROM station_data sd LEFT JOIN data dd ON sd.id_station = dd.id_station WHERE dd.id IS NULL) )"); conn.exec(R"( DELETE FROM station WHERE id IN ( SELECT p.id FROM station p LEFT JOIN data d ON d.id_station = p.id WHERE d.id is NULL) )"); } } } } } dballe-8.6/dballe/db/v7/trace.cc0000644000175000017500000001377013554564112013322 00000000000000#include "trace.h" #include "dballe/core/query.h" #include #include using namespace wreport; namespace dballe { namespace db { namespace v7 { namespace { std::string query_to_string(const Query& query) { std::stringstream json_buf; core::JSONWriter writer(json_buf); core::Query::downcast(query).serialize(writer); return json_buf.str(); } std::vector read_argv() { std::vector argv; FILE* in = fopen("/proc/self/cmdline", "rb"); if (!in) throw error_system("cannot open /proc/self/cmdline"); std::string cur; char c; while ((c = getc(in)) != EOF) { if (c == 0) { argv.push_back(cur); cur.clear(); } else cur += c; } if (ferror(in)) { int e = errno; fclose(in); throw error_system("cannot read from /proc/self/cmdline", e); } return argv; } std::string format_time(time_t time) { struct tm tmp; localtime_r(&time, &tmp); char buf[20]; strftime(buf, 20, "%Y-%m-%dT%H:%M:%S", &tmp); return buf; } std::string format_time_fname(time_t time) { struct tm tmp; localtime_r(&time, &tmp); char buf[20]; strftime(buf, 20, "%Y%m%d-%H%M%S", &tmp); return buf; } } namespace trace { Step::Step(const std::string& name) : name(name), start(clock()) { } Step::Step(const std::string& name, const std::string& detail) : name(name), detail(detail), start(clock()) { } Step::~Step() { delete child; delete sibling; } void Step::done() { end = clock(); } unsigned Step::elapsed_usec() const { return (end - start) * 1000000 / CLOCKS_PER_SEC; } void Step::to_json(core::JSONWriter& writer) const { writer.start_mapping(); writer.add("name", name); writer.add("detail", detail); writer.add("rows", (int)rows); writer.add("usecs", (int)elapsed_usec()); if (child) { writer.add("ops"); writer.start_list(); for (Step* s = child; s; s = s->sibling) s->to_json(writer); writer.end_list(); } writer.end_mapping(); } Tracer<> Transaction::trace_query_stations(const Query& query) { return Tracer<>(add_child(new trace::Step("query_stations", query_to_string(query)))); } Tracer<> Transaction::trace_query_station_data(const Query& query) { return Tracer<>(add_child(new trace::Step("query_station_data", query_to_string(query)))); } Tracer<> Transaction::trace_query_data(const Query& query) { return Tracer<>(add_child(new trace::Step("query_data", query_to_string(query)))); } Tracer<> Transaction::trace_query_summary(const Query& query) { return Tracer<>(add_child(new trace::Step("query_summary", query_to_string(query)))); } Tracer<> Transaction::trace_import(unsigned count) { return Tracer<>(add_child(new trace::Step("import", std::to_string(count)))); } Tracer<> Transaction::trace_export_msgs(const Query& query) { return Tracer<>(add_child(new trace::Step("export_msgs", query_to_string(query)))); } Tracer<> Transaction::trace_insert_station_data() { return Tracer<>(add_child(new trace::Step("insert_station_data"))); } Tracer<> Transaction::trace_insert_data() { return Tracer<>(add_child(new trace::Step("insert_data"))); } Tracer<> Transaction::trace_add_station_vars() { return Tracer<>(add_child(new trace::Step("insert_data"))); } Tracer<> Transaction::trace_func(const std::string& name) { return Tracer<>(add_child(new trace::Step(name))); } Tracer<> Transaction::trace_remove_station_data(const Query& query) { return Tracer<>(add_child(new trace::Step("remove_station_data", query_to_string(query)))); } Tracer<> Transaction::trace_remove_data(const Query& query) { return Tracer<>(add_child(new trace::Step("remove_data", query_to_string(query)))); } Tracer<> Transaction::trace_remove_station_data_by_id(int id) { return Tracer<>(add_child(new trace::Step("remove_station_data_by_id", std::to_string(id)))); } Tracer<> Transaction::trace_remove_data_by_id(int id) { return Tracer<>(add_child(new trace::Step("remove_data_by_id", std::to_string(id)))); } } QuietCollectTrace::~QuietCollectTrace() { for (auto& i: steps) delete i; } Tracer<> QuietCollectTrace::trace_connect(const std::string& url) { steps.push_back(new trace::Step("connect", url)); return Tracer<>(steps.back()); } Tracer<> QuietCollectTrace::trace_reset(const char* repinfo_file) { steps.push_back(new trace::Step("reset", repinfo_file ? repinfo_file : "")); return steps.back(); } Tracer QuietCollectTrace::trace_transaction() { trace::Transaction* res = new trace::Transaction; steps.push_back(res); return res; } Tracer<> QuietCollectTrace::trace_remove_all() { steps.push_back(new trace::Step("remove_all")); return Tracer<>(steps.back()); } Tracer<> QuietCollectTrace::trace_vacuum() { steps.push_back(new trace::Step("vacuum")); return Tracer<>(steps.back()); } CollectTrace::CollectTrace(const std::string& logdir) : logdir(logdir), start(time(nullptr)) { } void CollectTrace::save() { pid_t pid = getpid(); std::stringstream json_buf; core::JSONWriter writer(json_buf); writer.start_mapping(); writer.add("cmdline"); writer.add_list(read_argv()); writer.add("pid", (int)pid); writer.add("start", format_time(start)); writer.add("end", format_time(time(nullptr))); writer.add("ops"); writer.start_list(); for (const auto& s: steps) s->to_json(writer); writer.end_list(); writer.end_mapping(); std::string fname = logdir; fname += "/"; fname += format_time_fname(start); fname += "-"; fname += std::to_string(pid); fname += ".json"; FILE* out = fopen(fname.c_str(), "wt"); fwrite(json_buf.str().data(), json_buf.str().size(), 1, out); putc('\n', out); fclose(out); } static bool _in_test_suite = false; bool Trace::in_test_suite() { return _in_test_suite; } void Trace::set_in_test_suite() { _in_test_suite = true; } } } } dballe-8.6/dballe/db/v7/trace.h0000644000175000017500000001435613554564112013165 00000000000000#ifndef DBALLE_DB_V7_TRACE_H #define DBALLE_DB_V7_TRACE_H #include #include #include #include #include #include namespace dballe { namespace db { namespace v7 { namespace trace { struct Aggregate { unsigned count = 0; unsigned rows = 0; clock_t ticks = 0; }; /** * One operation being traced */ class Step { protected: /// Parent operation in the operation stack Step* parent = nullptr; /// First child operation in the operation stack Step* child = nullptr; /// Next sibling operation in the operation stack Step* sibling = nullptr; /// Operation name std::string name; /// Optional details about the operation std::string detail; /// Number of database rows affected unsigned rows = 0; /// Timing start clock_t start = 0; /// Timing end clock_t end = 0; template void add_sibling(T* step) { if (!sibling) { sibling = step; step->parent = parent; } else sibling->add_sibling(step); } Step* first_sibling(const std::string& name) { if (this->name == name) return this; if (!sibling) return nullptr; return sibling->first_sibling(name); } Step* last_sibling(const std::string& name, Step* last=nullptr) { if (this->name == name) { if (!sibling) return this; return sibling->last_sibling(name, this); } if (!sibling) return last; return sibling->last_sibling(name, last); } void _aggregate(const std::string& name, Aggregate& agg) { if (this->name == name) { ++agg.count; agg.rows += rows; agg.ticks += end - start; } if (sibling) sibling->_aggregate(name, agg); if (child) child->_aggregate(name, agg); } public: Step(const std::string& name); Step(const std::string& name, const std::string& detail); ~Step(); void done(); unsigned elapsed_usec() const; void to_json(core::JSONWriter& writer) const; // Remove all children accumulated so far void clear() { delete child; child = nullptr; } Aggregate aggregate(const std::string& name) { Aggregate res; if (child) child->_aggregate(name, res); return res; } Step* first_child(const std::string& name) { if (!child) return nullptr; return child->first_sibling(name); } Step* last_child(const std::string& name) { if (!child) return nullptr; return child->last_sibling(name); } void add_row(unsigned amount=1) { rows += amount; } template T* add_child(T* step) { if (!child) { child = step; step->parent = this; } else child->add_sibling(step); return step; } Step* trace_select(const std::string& query, unsigned rows=0) { Step* res = add_child(new Step("select", query)); res->rows = rows; return res; } Step* trace_insert(const std::string& query, unsigned rows=0) { Step* res = add_child(new Step("insert", query)); res->rows = rows; return res; } Step* trace_update(const std::string& query, unsigned rows=0) { Step* res = add_child(new Step("update", query)); res->rows = rows; return res; } Step* trace_delete(const std::string& query, unsigned rows=0) { Step* res = add_child(new Step("delete", query)); res->rows = rows; return res; } }; class Transaction : public Step { public: Transaction() : Step("transaction") {} Tracer<> trace_query_stations(const Query& query); Tracer<> trace_query_station_data(const Query& query); Tracer<> trace_query_data(const Query& query); Tracer<> trace_query_summary(const Query& query); Tracer<> trace_import(unsigned count); Tracer<> trace_export_msgs(const Query& query); Tracer<> trace_insert_station_data(); Tracer<> trace_insert_data(); Tracer<> trace_add_station_vars(); Tracer<> trace_func(const std::string& name); Tracer<> trace_remove_station_data(const Query& query); Tracer<> trace_remove_data(const Query& query); Tracer<> trace_remove_station_data_by_id(int id); Tracer<> trace_remove_data_by_id(int id); }; } struct Trace { virtual ~Trace() {} virtual Tracer<> trace_connect(const std::string& url) = 0; virtual Tracer<> trace_reset(const char* repinfo_file=0) = 0; virtual Tracer trace_transaction() = 0; virtual Tracer<> trace_remove_all() = 0; virtual Tracer<> trace_vacuum() = 0; virtual void save() = 0; static bool in_test_suite(); static void set_in_test_suite(); }; struct NullTrace : public Trace { Tracer<> trace_connect(const std::string& url) override { return Tracer<>(nullptr); } Tracer<> trace_reset(const char* repinfo_file=0) override { return Tracer<>(nullptr); } Tracer trace_transaction() override { return Tracer(nullptr); } Tracer<> trace_remove_all() override { return Tracer<>(nullptr); } Tracer<> trace_vacuum() override { return Tracer<>(nullptr); } void save() override {} }; class QuietCollectTrace : public Trace { protected: std::vector steps; public: QuietCollectTrace() = default; QuietCollectTrace(const QuietCollectTrace&) = delete; QuietCollectTrace(QuietCollectTrace&&) = delete; QuietCollectTrace& operator=(const QuietCollectTrace&) = delete; QuietCollectTrace& operator=(QuietCollectTrace&&) = delete; ~QuietCollectTrace(); Tracer<> trace_connect(const std::string& url) override; Tracer<> trace_reset(const char* repinfo_file=0) override; Tracer trace_transaction() override; Tracer<> trace_remove_all() override; Tracer<> trace_vacuum() override; void save() override {} }; class CollectTrace : public QuietCollectTrace { protected: std::string logdir; time_t start; public: CollectTrace(const std::string& logdir); void save() override; }; } } } #endif dballe-8.6/dballe/db/v7/station-test.cc0000644000175000017500000000341313554564112014653 00000000000000#include "dballe/db/tests.h" #include "dballe/sql/sql.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/fwd.h" #include "driver.h" #include "repinfo.h" #include "station.h" #include "transaction.h" #include "config.h" using namespace dballe; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { class Tests : public FixtureTestCase> { using FixtureTestCase::FixtureTestCase; typedef EmptyTransactionFixture Fixture; void register_tests() override; }; Tests test_sqlite("db_v7_station_sqlite", "SQLITE"); #ifdef HAVE_LIBPQ Tests test_psql("db_v7_station_postgresql", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests test_mysql("db_v7_station_mysql", "MYSQL"); #endif void Tests::register_tests() { add_method("insert", [](Fixture& f) { db::v7::Tracer<> trc; // Insert some values and try to read them again auto& st = f.tr->station(); dballe::DBStation sde1; dballe::DBStation sde2; int si; // Insert a mobile station sde1.report = "synop"; sde1.coords = Coords(4500000, 1100000); sde1.ident = "ciao"; si = st.maybe_get_id(trc, sde1); wassert(actual(si) == MISSING_INT); // Create si = st.insert_new(trc, sde1); wassert(actual(si) == 1); // Insert a fixed station sde2.report = "synop"; sde2.coords = Coords(4600000, 1200000); sde2.ident = nullptr; si = st.maybe_get_id(trc, sde2); wassert(actual(si) == MISSING_INT); // Create si = st.insert_new(trc, sde2); wassert(actual(si) == 2); // Get the ID of the first station si = st.maybe_get_id(trc, sde1); wassert(actual(si) == 1); // Get the ID of the second station si = st.maybe_get_id(trc, sde2); wassert(actual(si) == 2); }); } } dballe-8.6/dballe/db/v7/qbuilder.h0000644000175000017500000000731213554564112013670 00000000000000#ifndef DBA_DB_V7_QBUILDER_H #define DBA_DB_V7_QBUILDER_H #include #include #include #include namespace dballe { struct Varmatch; namespace db { namespace v7 { /// Build SQL queries for V7 databases struct QueryBuilder { dballe::sql::Connection& conn; /** Database to operate on */ std::shared_ptr tr; /** * If defined, it need to point to the identifier to be used as the only * bound input parameter. * * If not defined, there are no bound input parameters in this query */ const char* bind_in_ident = nullptr; bool select_station = false; // ana_id, lat, lon, ident bool select_varinfo = false; // rep_cod, id_ltr, varcode // IdQuery bool select_data_id = false; // id_data // DataQuery bool select_data = false; // datetime, value // SummaryQuery bool select_summary_details = false; // id_data, datetime, datetimemax /// Query object const core::Query& query; /** Dynamically generated SQL query */ dballe::sql::Querybuf sql_query; /// FROM part of the SQL query dballe::sql::Querybuf sql_from; /// WHERE part of the SQL query dballe::sql::Querybuf sql_where; /// Modifier flags to enable special query behaviours const unsigned int modifiers; /// True if we are querying station information, rather than measured data bool query_station_vars; QueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars); virtual ~QueryBuilder() {} void build(); protected: // Add WHERE conditions bool add_pa_where(const char* tbl); bool add_dt_where(const char* tbl); bool add_ltr_where(const char* tbl); bool add_varcode_where(const char* tbl); bool add_repinfo_where(const char* tbl); bool add_datafilter_where(const char* tbl); virtual void build_select() = 0; virtual bool build_where() = 0; virtual void build_order_by() = 0; }; struct StationQueryBuilder : public QueryBuilder { StationQueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers) : QueryBuilder(tr, query, modifiers, false) {} virtual void build_select(); virtual bool build_where(); virtual void build_order_by(); }; struct DataQueryBuilder : public QueryBuilder { /// Attribute filter, if requested Varmatch* attr_filter = nullptr; /// True if we also query attributes of data bool query_attrs; /// True if the select includes the attrs field bool select_attrs = false; DataQueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars); ~DataQueryBuilder(); // bool add_attrfilter_where(const char* tbl); /// Match the attributes of var against attr_filter bool match_attrs(const wreport::Var& var) const; virtual void build_select(); virtual bool build_where(); virtual void build_order_by(); }; struct IdQueryBuilder : public DataQueryBuilder { IdQueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars) : DataQueryBuilder(tr, query, modifiers, query_station_vars) {} virtual void build_select(); virtual void build_order_by(); }; struct SummaryQueryBuilder : public DataQueryBuilder { SummaryQueryBuilder(std::shared_ptr tr, const core::Query& query, unsigned int modifiers, bool query_station_vars) : DataQueryBuilder(tr, query, modifiers, query_station_vars) {} virtual void build_select(); virtual void build_order_by(); }; } } } #endif dballe-8.6/dballe/db/v7/driver.cc0000644000175000017500000000432113554564112013507 00000000000000#include "driver.h" #include "config.h" #include "dballe/db/v7/sqlite/driver.h" #include "dballe/sql/sqlite.h" #ifdef HAVE_LIBPQ #include "dballe/db/v7/postgresql/driver.h" #include "dballe/sql/postgresql.h" #endif #ifdef HAVE_MYSQL #include "dballe/db/v7/mysql/driver.h" #include "dballe/sql/mysql.h" #endif #include #include using namespace wreport; using namespace std; namespace dballe { namespace db { namespace v7 { Driver::Driver(sql::Connection& connection) : connection(connection) { } Driver::~Driver() { } void Driver::create_tables(db::Format format) { switch (format) { case Format::V7: create_tables_v7(); break; default: throw wreport::error_consistency("cannot create tables on the given DB format"); } } void Driver::delete_tables(db::Format format) { switch (format) { case Format::V7: delete_tables_v7(); break; default: throw wreport::error_consistency("cannot delete tables on the given DB format"); } } void Driver::remove_all(db::Format format) { switch (format) { case Format::V7: remove_all_v7(); break; default: throw wreport::error_consistency("cannot empty a database with the given format"); } } void Driver::remove_all_v7() { connection.execute("DELETE FROM station_data"); connection.execute("DELETE FROM data"); connection.execute("DELETE FROM levtr"); connection.execute("DELETE FROM station"); } std::unique_ptr Driver::create(dballe::sql::Connection& conn) { using namespace dballe::sql; if (SQLiteConnection* c = dynamic_cast(&conn)) return unique_ptr(new sqlite::Driver(*c)); #ifdef HAVE_LIBPQ else if (PostgreSQLConnection* c = dynamic_cast(&conn)) return unique_ptr(new postgresql::Driver(*c)); #endif #ifdef HAVE_MYSQL else if (MySQLConnection* c = dynamic_cast(&conn)) return unique_ptr(new mysql::Driver(*c)); #endif else throw error_unimplemented("DB drivers only implemented for " #ifdef HAVE_LIBPQ "PostgreSQL, " #endif #ifdef HAVE_MYSQL "MySQL, " #endif "SQLite connectors"); } } } } dballe-8.6/dballe/db/fwd.h0000644000175000017500000000032213554564112012277 00000000000000#ifndef DBALLE_DB_FWD_H #define DBALLE_DB_FWD_H namespace dballe { namespace db { class CursorStation; class CursorStationData; class CursorData; class CursorSummary; class DB; class Transaction; } } #endif dballe-8.6/dballe/db/explorer.h0000644000175000017500000000645313554564112013372 00000000000000#ifndef DBALLE_DB_EXPLORER_H #define DBALLE_DB_EXPLORER_H #include #include #include #include #include #include namespace dballe { namespace db { template class BaseExplorer { protected: /// Summary of the whole database dballe::db::BaseSummary* _global_summary = nullptr; /// Currently active filter dballe::core::Query filter; /// Summary of active_filter dballe::db::BaseSummary* _active_summary = nullptr; /// Regenerate _active_summary based on filter void update_active_summary(); public: class Update { protected: BaseExplorer* explorer = nullptr; Update(BaseExplorer* explorer); public: Update(); Update(const Update&) = delete; Update(Update&&); ~Update(); Update& operator=(const Update&) = delete; Update& operator=(Update&&); /// Merge summary data from a database void add_db(dballe::db::Transaction& tr); /// Merge summary data from a database void add_cursor(dballe::CursorSummary& cur); /// Load the explorer contents from JSON void add_json(core::json::Stream& in); /// Merge the contents of another explorer into this one template void add_explorer(const BaseExplorer& explorer); /// Merge the contents of a message void add_message(const dballe::Message& message); /// Merge the contents of a vector of messages void add_messages(const std::vector>& messages); void commit(); friend class BaseExplorer; }; BaseExplorer(); BaseExplorer(const BaseExplorer&) = delete; BaseExplorer(BaseExplorer&&) = delete; ~BaseExplorer(); BaseExplorer& operator=(const BaseExplorer&) = delete; BaseExplorer& operator=(BaseExplorer&&) = delete; /// Get the current filter const dballe::Query& get_filter() const; /** * Set a new filter, updating all browsing data */ void set_filter(const dballe::Query& query); /** * Throw away all cached data and rebuild the explorer from scratch. * * Use this when you suspect that the database has been externally modified */ Update rebuild(); /** * Merge new data into the explorer */ Update update(); /// Get a reference to the global summary const dballe::db::BaseSummary& global_summary() const; /// Get a reference to the summary for the current filter const dballe::db::BaseSummary& active_summary() const; /// Export the explorer to JSON void to_json(core::JSONWriter& writer) const; }; /** * Explorer without database station IDs */ typedef BaseExplorer Explorer; /** * Explorer with database station IDs */ typedef BaseExplorer DBExplorer; extern template class BaseExplorer; extern template void BaseExplorer::Update::add_explorer(const BaseExplorer&); extern template class BaseExplorer; extern template void BaseExplorer::Update::add_explorer(const BaseExplorer&); } } #endif dballe-8.6/dballe/db/explorer-test.cc0000644000175000017500000001556713554564112014513 00000000000000#define _DBALLE_TEST_CODE #include "dballe/db/tests.h" #include "dballe/db/v7/db.h" #include "dballe/db/v7/transaction.h" #include "explorer.h" #include "config.h" using namespace dballe; using namespace dballe::db; using namespace dballe::tests; using namespace wreport; using namespace std; namespace { template class Tests : public FixtureTestCase> { typedef EmptyTransactionFixture Fixture; using FixtureTestCase::FixtureTestCase; void register_tests() override; }; Tests tg1("db_explorer_v7_sqlite_explorer", "SQLITE"); Tests tg2("db_explorer_v7_sqlite_dbexplorer", "SQLITE"); #ifdef HAVE_LIBPQ Tests tg3("db_explorer_v7_postgresql_explorer", "POSTGRESQL"); Tests tg4("db_explorer_v7_postgresql_dbexplorer", "POSTGRESQL"); #endif #ifdef HAVE_MYSQL Tests tg5("db_explorer_v7_mysql_explorer", "MYSQL"); Tests tg6("db_explorer_v7_mysql_dbexplorer", "MYSQL"); #endif template void test_explorer_contents(const BaseExplorer& explorer) { vector stations; stations.clear(); for (const auto& s: explorer.global_summary().stations()) stations.push_back(s.station); wassert(actual(stations.size()) == 2); wassert(actual(stations[0].report) == "metar"); wassert(actual(stations[1].report) == "synop"); stations.clear(); for (const auto& s: explorer.active_summary().stations()) stations.push_back(s.station); wassert(actual(stations.size()) == 1); wassert(actual(stations[0].report) == "metar"); vector reports; reports.clear(); for (const auto& v: explorer.global_summary().reports()) reports.push_back(v); wassert(actual(reports.size()) == 2); wassert(actual(reports[0]) == "metar"); wassert(actual(reports[1]) == "synop"); reports.clear(); for (const auto& v: explorer.active_summary().reports()) reports.push_back(v); wassert(actual(reports.size()) == 1); wassert(actual(reports[0]) == "metar"); vector levels; levels.clear(); for (const auto& v: explorer.global_summary().levels()) levels.push_back(v); wassert(actual(levels.size()) == 1); wassert(actual(levels[0]) == Level(10, 11, 15, 22)); levels.clear(); for (const auto& v: explorer.active_summary().levels()) levels.push_back(v); wassert(actual(levels.size()) == 1); wassert(actual(levels[0]) == Level(10, 11, 15, 22)); vector tranges; tranges.clear(); for (const auto& v: explorer.global_summary().tranges()) tranges.push_back(v); wassert(actual(tranges.size()) == 2); wassert(actual(tranges[0]) == Trange(20, 111, 122)); wassert(actual(tranges[1]) == Trange(20, 111, 123)); tranges.clear(); for (const auto& v: explorer.active_summary().tranges()) tranges.push_back(v); wassert(actual(tranges.size()) == 1); wassert(actual(tranges[0]) == Trange(20, 111, 123)); vector vars; vars.clear(); for (const auto& v: explorer.global_summary().varcodes()) vars.push_back(v); wassert(actual(vars.size()) == 2); wassert(actual(vars[0]) == WR_VAR(0, 1, 11)); wassert(actual(vars[1]) == WR_VAR(0, 1, 12)); vars.clear(); for (const auto& v: explorer.active_summary().varcodes()) vars.push_back(v); wassert(actual(vars.size()) == 2); wassert(actual(vars[0]) == WR_VAR(0, 1, 11)); wassert(actual(vars[1]) == WR_VAR(0, 1, 12)); } template void Tests::register_tests() { this->add_method("filter_rep_memo", [](Fixture& f) { // Test building a summary and checking if it supports queries OldDballeTestDataSet test_data; wassert(f.populate(test_data)); EXPLORER explorer; { auto update = explorer.rebuild(); wassert(update.add_db(*f.tr)); } core::Query query; query.set_from_test_string("rep_memo=metar"); explorer.set_filter(query); wassert(test_explorer_contents(explorer)); wassert(actual(explorer.global_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 0)); wassert(actual(explorer.global_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer.global_summary().data_count()) == 4u); wassert(actual(explorer.active_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer.active_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer.active_summary().data_count()) == 2u); std::stringstream json; core::JSONWriter writer(json); explorer.to_json(writer); wassert(actual(json.str()).startswith("{\"summary\":{")); json.seekg(0); core::json::Stream in(json); EXPLORER explorer1; { auto update = explorer1.update(); wassert(update.add_json(in)); } explorer1.set_filter(query); wassert(test_explorer_contents(explorer1)); wassert(actual(explorer1.global_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 0)); wassert(actual(explorer1.global_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.global_summary().data_count()) == 4u); wassert(actual(explorer1.active_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.active_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.active_summary().data_count()) == 2u); }); this->add_method("merge", [](Fixture& f) { OldDballeTestDataSet test_data; wassert(f.populate(test_data)); EXPLORER explorer; { auto update = explorer.rebuild(); wassert(update.add_db(*f.tr)); } EXPLORER explorer1; core::Query query; query.set_from_test_string("rep_memo=metar"); explorer1.set_filter(query); { auto update = explorer1.update(); wassert(update.add_explorer(explorer)); wassert(update.add_explorer(explorer)); } wassert(test_explorer_contents(explorer1)); wassert(actual(explorer1.global_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 0)); wassert(actual(explorer1.global_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.global_summary().data_count()) == 8u); wassert(actual(explorer1.active_summary().datetime_min()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.active_summary().datetime_max()) == Datetime(1945, 4, 25, 8, 30)); wassert(actual(explorer1.active_summary().data_count()) == 4u); }); this->add_method("merge_self", [](Fixture& f) { EXPLORER explorer; { auto update = explorer.update(); auto e = wassert_throws(wreport::error_consistency, update.add_explorer(explorer)); wassert(actual(e.what()) == "Adding an Explorer to itself is not supported"); } }); } } dballe-8.6/dballe/db/summary.h0000644000175000017500000003217013572414716013226 00000000000000#ifndef DBALLE_DB_SUMMARY_H #define DBALLE_DB_SUMMARY_H #include #include #include #include #include #include #include #include namespace dballe { namespace db { namespace summary { /** * Description of a variable, independent of where and when it was measured */ struct VarDesc { dballe::Level level; dballe::Trange trange; wreport::Varcode varcode; VarDesc() = default; VarDesc(const dballe::Level& level, const dballe::Trange& trange, wreport::Varcode varcode) : level(level), trange(trange), varcode(varcode) {} VarDesc(const VarDesc&) = default; bool operator==(const VarDesc& o) const { return std::tie(level, trange, varcode) == std::tie(o.level, o.trange, o.varcode); } bool operator!=(const VarDesc& o) const { return std::tie(level, trange, varcode) != std::tie(o.level, o.trange, o.varcode); } bool operator< (const VarDesc& o) const { return std::tie(level, trange, varcode) < std::tie(o.level, o.trange, o.varcode); } bool operator<=(const VarDesc& o) const { return std::tie(level, trange, varcode) <= std::tie(o.level, o.trange, o.varcode); } bool operator> (const VarDesc& o) const { return std::tie(level, trange, varcode) > std::tie(o.level, o.trange, o.varcode); } bool operator>=(const VarDesc& o) const { return std::tie(level, trange, varcode) >= std::tie(o.level, o.trange, o.varcode); } }; /** * Statistics about a variable */ struct VarEntry { VarDesc var; dballe::DatetimeRange dtrange; size_t count = 0; VarEntry() = default; VarEntry(const VarDesc& var, const dballe::DatetimeRange& dtrange, size_t count) : var(var), dtrange(dtrange), count(count) { } VarEntry(const VarEntry&) = default; bool operator==(const VarEntry& o) const { return std::tie(var, dtrange, count) == std::tie(o.var, o.dtrange, o.count); } bool operator!=(const VarEntry& o) const { return std::tie(var, dtrange, count) != std::tie(o.var, o.dtrange, o.count); } void merge(const dballe::DatetimeRange& dtrange, size_t count) { this->dtrange.merge(dtrange); this->count += count; } void to_json(core::JSONWriter& writer) const; static VarEntry from_json(core::json::Stream& in); DBALLE_TEST_ONLY void dump(FILE* out) const; }; inline const VarDesc& station_entry_get_value(const VarEntry& item) { return item.var; } /** * Information about a station, and statistics about its variables. * * It behaves similarly to a std::vector */ template struct StationEntry : protected core::SmallSet { using SmallSet::iterator; using SmallSet::const_iterator; using SmallSet::reverse_iterator; using SmallSet::const_reverse_iterator; using SmallSet::begin; using SmallSet::end; using SmallSet::rbegin; using SmallSet::rend; using SmallSet::size; using SmallSet::empty; using SmallSet::add; bool operator==(const StationEntry& o) const { return SmallSet::operator==(o); } bool operator!=(const StationEntry& o) const { return SmallSet::operator!=(o); } Station station; StationEntry() = default; template StationEntry(const Station& station, const StationEntry& entry) : station(station) { for (const auto& item: entry) this->add(item); } StationEntry(const Station& station, const VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count) : station(station) { add(vd, dtrange, count); } StationEntry(const StationEntry& entries, const dballe::Query& query) : station(entries.station) { add_filtered(entries, query); } StationEntry(const StationEntry&) = default; void add(const VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count); template void add(const StationEntry& entries); void add_filtered(const StationEntry& entries, const dballe::Query& query); void to_json(core::JSONWriter& writer) const; static StationEntry from_json(core::json::Stream& in); DBALLE_TEST_ONLY void dump(FILE* out) const; }; template inline const Station& station_entries_get_value(const StationEntry& item) { return item.station; } /** * Index of all stations known to a summary * * It behaves similarly to a std::vector> */ template struct StationEntries : protected core::SmallSet, Station, station_entries_get_value> { typedef core::SmallSet, Station, station_entries_get_value> Parent; typedef typename Parent::iterator iterator; typedef typename Parent::const_iterator const_iterator; typedef typename Parent::reverse_iterator reverse_iterator; typedef typename Parent::const_reverse_iterator const_reverse_iterator; using Parent::begin; using Parent::end; using Parent::rbegin; using Parent::rend; using Parent::size; using Parent::empty; bool operator==(const StationEntries& o) const { return Parent::operator==(o); } bool operator!=(const StationEntries& o) const { return Parent::operator!=(o); } /// Merge the given entry void add(const Station& station, const VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count); /// Merge the given entries template void add(const StationEntries& entry); /// Merge the given entries void add(const StationEntries& entry); /// Merge the given entry void add(const StationEntry& entry); void add_filtered(const StationEntries& entry, const dballe::Query& query); bool has(const Station& station) const { return this->find(station) != this->end(); } const StationEntries& sorted() const { if (this->dirty) this->rearrange_dirty(); return *this; } }; template struct Cursor : public impl::CursorSummary { struct Entry { const summary::StationEntry& station_entry; const summary::VarEntry& var_entry; Entry(const summary::StationEntry& station_entry, const summary::VarEntry& var_entry) : station_entry(station_entry), var_entry(var_entry) {} }; std::vector results; typename std::vector::const_iterator cur; bool at_start = true; Cursor(const summary::StationEntries& entries, const Query& query); bool has_value() const { return !at_start && cur != results.end(); } int remaining() const override { if (at_start) return results.size(); return results.end() - cur; } bool next() override { if (at_start) { cur = results.begin(); at_start = false; } else if (cur != results.end()) ++cur; return cur != results.end(); } void discard() override { cur = results.end(); } static DBStation _get_dbstation(const DBStation& s) { return s; } static DBStation _get_dbstation(const dballe::Station& station) { DBStation res; res.report = station.report; res.coords = station.coords; res.ident = station.ident; return res; } static int _get_station_id(const DBStation& s) { return s.id; } static int _get_station_id(const dballe::Station& s) { return MISSING_INT; } DBStation get_station() const override { return _get_dbstation(cur->station_entry.station); } #if 0 int get_station_id() const override { return _get_station_id(cur->station_entry.station); } Coords get_coords() const override { return cur->station_entry.station.coords; } Ident get_ident() const override { return cur->station_entry.station.ident; } std::string get_report() const override { return cur->station_entry.station.report; } unsigned test_iterate(FILE* dump=0) override { unsigned count; for (count = 0; next(); ++count) ; #if 0 if (dump) cur->dump(dump); #endif return count; } #endif Level get_level() const override { return cur->var_entry.var.level; } Trange get_trange() const override { return cur->var_entry.var.trange; } wreport::Varcode get_varcode() const override { return cur->var_entry.var.varcode; } DatetimeRange get_datetimerange() const override { return cur->var_entry.dtrange; } size_t get_count() const override { return cur->var_entry.count; } void enq(impl::Enq& enq) const; }; extern template class Cursor; extern template class Cursor; } /** * High level objects for working with DB-All.e DB summaries */ template class BaseSummary { protected: // Summary of items for the currently active filter summary::StationEntries entries; mutable core::SortedSmallUniqueValueSet m_reports; mutable core::SortedSmallUniqueValueSet m_levels; mutable core::SortedSmallUniqueValueSet m_tranges; mutable core::SortedSmallUniqueValueSet m_varcodes; mutable dballe::DatetimeRange dtrange; mutable size_t count = 0; mutable bool dirty = false; void recompute_summaries() const; public: BaseSummary(); BaseSummary(const BaseSummary&) = delete; BaseSummary(BaseSummary&&) = delete; BaseSummary& operator=(const BaseSummary&) = delete; BaseSummary& operator=(BaseSummary&&) = delete; bool operator==(const BaseSummary& o) const { return entries == o.entries; } const summary::StationEntries& stations() const { if (dirty) recompute_summaries(); return entries.sorted(); } const core::SortedSmallUniqueValueSet& reports() const { if (dirty) recompute_summaries(); return m_reports; } const core::SortedSmallUniqueValueSet& levels() const { if (dirty) recompute_summaries(); return m_levels; } const core::SortedSmallUniqueValueSet& tranges() const { if (dirty) recompute_summaries(); return m_tranges; } const core::SortedSmallUniqueValueSet& varcodes() const { if (dirty) recompute_summaries(); return m_varcodes; } /** * Recompute reports, levels, tranges, and varcodes. * * Call this after performing changes to the summary, to make those sets * valid before reading them. */ const Datetime& datetime_min() const { if (dirty) recompute_summaries(); return dtrange.min; } const Datetime& datetime_max() const { if (dirty) recompute_summaries(); return dtrange.max; } unsigned data_count() const { if (dirty) recompute_summaries(); return count; } /** * Query the contents of the summary * * @param query * The record with the query data (see technical specifications, par. 1.6.4 * "parameter output/input") * @return * The cursor to use to iterate over the results. The results are the * same as DB::query_summary. */ std::unique_ptr query_summary(const Query& query) const; /// Add an entry to the summary void add(const Station& station, const summary::VarDesc& vd, const dballe::DatetimeRange& dtrange, size_t count); /// Add an entry to the summary taken from the current status of \a cur void add_cursor(const dballe::CursorSummary& cur); /// Add the contents of a Message void add_message(const dballe::Message& message); /// Add the contents of a Messages void add_messages(const std::vector>& messages); /// Merge the copy of another summary into this one template void add_summary(const BaseSummary& summary); /// Merge the copy of another summary into this one void add_filtered(const BaseSummary& summary, const dballe::Query& query); #if 0 /** * Merge entries with duplicate metadata * * Call this function if you call add_entry with entries that may already * exist in the summary */ void merge_entries(); #endif /// Serialize to JSON void to_json(core::JSONWriter& writer) const; /// Load contents from JSON, merging with the current contents void load_json(core::json::Stream& in); DBALLE_TEST_ONLY void dump(FILE* out) const; }; /** * Summary without database station IDs */ typedef BaseSummary Summary; /** * Summary with database station IDs */ typedef BaseSummary DBSummary; extern template class BaseSummary; extern template void BaseSummary::add_summary(const BaseSummary&); extern template void BaseSummary::add_summary(const BaseSummary&); extern template class BaseSummary; extern template void BaseSummary::add_summary(const BaseSummary&); extern template void BaseSummary::add_summary(const BaseSummary&); } } #endif dballe-8.6/dballe/db/tests.cc0000644000175000017500000002075413554564112013032 00000000000000#include "tests.h" #include "v7/db.h" #include "v7/transaction.h" #include "v7/driver.h" #include "dballe/sql/sql.h" #include "dballe/sql/sqlite.h" #include #include #include #include #include #include "config.h" using namespace wreport; using namespace std; namespace dballe { namespace tests { impl::Messages messages_from_db(std::shared_ptr tr, const dballe::Query& query) { impl::Messages res; auto cursor = tr->query_messages(query); while (cursor->next()) res.emplace_back(cursor->detach_message()); return res; } impl::Messages messages_from_db(std::shared_ptr tr, const char* query) { impl::Messages res; auto cursor = tr->query_messages(*dballe::tests::query_from_string(query)); while (cursor->next()) res.emplace_back(cursor->detach_message()); return res; } void ActualCursor::station_keys_match(const DBStation& expected) { wassert(actual(_actual.get_station()) == expected); } void ActualCursor::data_context_matches(const Data& expected) { db::CursorData* c = dynamic_cast(&_actual); if (!c) throw TestFailed("cursor is not an instance of CursorData"); const core::Data& exp = core::Data::downcast(expected); wassert(actual(_actual).station_keys_match(exp.station)); wassert(actual(c->get_level()) == exp.level); wassert(actual(c->get_trange()) == exp.trange); wassert(actual(c->get_datetime()) == exp.datetime); wassert(actual(c->get_station().report) == exp.station.report); wassert(actual(c->get_level()) == exp.level); wassert(actual(c->get_trange()) == exp.trange); wassert(actual(c->get_datetime()) == exp.datetime); } void ActualCursor::data_var_matches(const wreport::Var& expected) { if (db::CursorStationData* c = dynamic_cast(&_actual)) { wassert(actual(c->get_varcode()) == expected.code()); wassert(actual(c->get_var()) == expected); } else if (db::CursorData* c = dynamic_cast(&_actual)) { wassert(actual(c->get_varcode()) == expected.code()); wassert(actual(c->get_var()) == expected); } else throw TestFailed("cursor is not an instance of CursorValue"); } void ActualCursor::data_matches(const Data& ds, wreport::Varcode code) { wassert(actual(_actual).data_context_matches(ds)); wassert(actual(_actual).data_var_matches(ds, code)); } template void ActualDB::try_data_query(const std::string& query, unsigned expected) { core::Query q; q.set_from_test_string(query); try_data_query(q, expected); } template void ActualDB::try_data_query(const Query& query, unsigned expected) { // Run the query unique_ptr cur = this->_actual->query_data(query); // Check the number of results wassert(actual(cur->remaining()) == expected); unsigned count = dynamic_cast(cur.get())->test_iterate(/* stderr */); wassert(actual(count) == expected); } template void ActualDB::try_station_query(const std::string& query, unsigned expected) { // Run the query unique_ptr cur = this->_actual->query_stations(core_query_from_string(query)); // Check the number of results wassert(actual(cur->remaining()) == expected); unsigned count = dynamic_cast(cur.get())->test_iterate(/* stderr */); wassert(actual(count) == expected); } template void ActualDB::try_summary_query(const std::string& query, unsigned expected, result_checker check_results) { // Run the query auto cur = this->_actual->query_summary(core_query_from_string(query)); // Check the number of results // query_summary counts results in advance only optionally if (cur->remaining() != 0) wassert(actual(cur->remaining()) == expected); db::DBSummary summary; unsigned found = 0; while (cur->next()) { ++found; summary.add_cursor(*cur); } wassert(actual(found) == expected); if (check_results) wassert(check_results(summary)); } bool has_driver(const std::string& backend) { std::string envname = "DBA_DB"; if (!backend.empty()) { envname = "DBA_DB_"; envname += backend; } return getenv(envname.c_str()) != NULL; } template BaseDBFixture::BaseDBFixture(const char* backend) : backend(backend ? backend : "") { } template BaseDBFixture::~BaseDBFixture() { if (db && getenv("PAUSE") == nullptr) db->disappear(); } template bool BaseDBFixture::has_driver() { return tests::has_driver(backend); } template void BaseDBFixture::create_db() { db = DB::create_db(backend, true); /* if (auto d = dynamic_cast(db.get())) if (auto c = dynamic_cast(d->conn)) c->trace(); */ } template void BaseDBFixture::test_setup() { Fixture::test_setup(); if (!has_driver()) throw TestSkipped(); if (!db) create_db(); } template void EmptyTransactionFixture::populate(TestDataSet& data_set) { wassert(data_set.populate_transaction(*tr)); } template void EmptyTransactionFixture::test_setup() { BaseDBFixture::test_setup(); tr = dynamic_pointer_cast(this->db->test_transaction()); } template void EmptyTransactionFixture::test_teardown() { if (tr) tr->rollback(); tr.reset(); BaseDBFixture::test_teardown(); } template void DBFixture::test_setup() { BaseDBFixture::test_setup(); auto tr = dynamic_pointer_cast(this->db->transaction()); tr->remove_all(); int added, deleted, updated; tr->update_repinfo(nullptr, &added, &deleted, &updated); tr->commit(); } template void DBFixture::populate_database(TestDataSet& data_set) { wassert(data_set.populate_db(*this->db)); } std::shared_ptr V7DB::create_db(const std::string& backend, bool wipe) { auto options = DBConnectOptions::test_create(backend.c_str()); options->wipe = wipe; return std::dynamic_pointer_cast(dballe::DB::connect(*options)); } void TestDataSet::populate_db(DB& db) { auto tr = db.transaction(); populate_transaction(*tr); tr->commit(); } void TestDataSet::populate_transaction(Transaction& tr) { // TODO: do everything in a single batch impl::DBInsertOptions opts; opts.can_replace = true; opts.can_add_stations = true; for (auto& d: stations) wassert(tr.insert_station_data(d.second, opts)); for (auto& d: data) wassert(tr.insert_data(d.second, opts)); } OldDballeTestDataSet::OldDballeTestDataSet() { stations["synop"].station.report = "synop"; stations["synop"].station.coords = Coords(12.34560, 76.54320); stations["synop"].values.set("B07030", 42); // Height stations["synop"].values.set("B07031", 234); // Heightbaro stations["synop"].values.set("B01001", 1); // Block stations["synop"].values.set("B01002", 52); // Station stations["synop"].values.set("B01019", "Cippo Lippo"); // Name stations["metar"] = stations["synop"]; stations["metar"].station.report = "metar"; data["synop"].station = stations["synop"].station; data["synop"].level = Level(10, 11, 15, 22); data["synop"].trange = Trange(20, 111, 122); data["synop"].datetime = Datetime(1945, 4, 25, 8); data["synop"].values.set("B01011", "DB-All.e!"); data["synop"].values.set("B01012", 300); data["metar"].station = stations["metar"].station; data["metar"].level = Level(10, 11, 15, 22); data["metar"].trange = Trange(20, 111, 123); data["metar"].datetime = Datetime(1945, 4, 25, 8, 30); data["metar"].values.set("B01011", "Arpa-Sim!"); data["metar"].values.set("B01012", 400); } ActualDB actual(std::shared_ptr actual) { return ActualDB(std::static_pointer_cast(actual)); } ActualDB actual(std::shared_ptr actual) { return ActualDB(std::static_pointer_cast(actual)); } template class BaseDBFixture; template class DBFixture; template class EmptyTransactionFixture; template class ActualDB; template class ActualDB; } } dballe-8.6/dballe/db/summary-access.in.cc0000644000175000017500000001014413554564112015221 00000000000000#include "summary.h" #include using namespace wreport; namespace dballe { namespace db { namespace summary { namespace { inline int get_station_id(const Station& station) { return MISSING_INT; } inline int get_station_id(const DBStation& station) { return station.id; } } /* * Summary */ template void Cursor::enq(impl::Enq& enq) const { const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": return; case "rep_memo": enq.set_string(cur->station_entry.station.report); case "report": enq.set_string(cur->station_entry.station.report); case "ana_id": enq.set_dballe_int(get_station_id(cur->station_entry.station)); case "mobile": enq.set_bool(!cur->station_entry.station.ident.is_missing()); case "ident": enq.set_ident(cur->station_entry.station.ident); case "lat": enq.set_lat(cur->station_entry.station.coords.lat); case "lon": enq.set_lon(cur->station_entry.station.coords.lon); case "coords": enq.set_coords(cur->station_entry.station.coords); case "station": enq.set_station(cur->station_entry.station); case "datetimemax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_datetime(cur->var_entry.dtrange.max.year); case "datetimemin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_datetime(cur->var_entry.dtrange.min.year); case "yearmax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.year); case "yearmin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.year); case "monthmax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.month); case "monthmin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.month); case "daymax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.day); case "daymin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.day); case "hourmax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.hour); case "hourmin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.hour); case "minumax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.minute); case "minumin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.minute); case "secmax": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.max.second); case "secmin": if (cur->var_entry.dtrange.is_missing()) return; else enq.set_int(cur->var_entry.dtrange.min.second); case "level": enq.set_level(cur->var_entry.var.level); case "leveltype1": enq.set_dballe_int(cur->var_entry.var.level.ltype1); case "l1": enq.set_dballe_int(cur->var_entry.var.level.l1); case "leveltype2": enq.set_dballe_int(cur->var_entry.var.level.ltype2); case "l2": enq.set_dballe_int(cur->var_entry.var.level.l2); case "trange": enq.set_trange(cur->var_entry.var.trange); case "pindicator": enq.set_dballe_int(cur->var_entry.var.trange.pind); case "p1": enq.set_dballe_int(cur->var_entry.var.trange.p1); case "p2": enq.set_dballe_int(cur->var_entry.var.trange.p2); case "var": enq.set_varcode(cur->var_entry.var.varcode); case "context_id": enq.set_int(cur->var_entry.count); case "count": enq.set_int(cur->var_entry.count); default: wreport::error_notfound::throwf("key %s not found on this query result", key); } } template void Cursor::enq(impl::Enq& enq) const; template void Cursor::enq(impl::Enq& enq) const; } } } dballe-8.6/dballe/cursor-test.cc0000644000175000017500000000046113554564112013566 00000000000000#include "dballe/core/tests.h" #include using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("cursor"); void Tests::register_tests() { add_method("empty", []{ }); } } dballe-8.6/dballe/msg/0000755000175000017500000000000013602152021011614 500000000000000dballe-8.6/dballe/msg/context-test.cc0000644000175000017500000001107413554564112014525 00000000000000#include "dballe/msg/tests.h" #include "context.h" #include using namespace wreport; using namespace dballe; using namespace dballe::tests; using namespace std; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("compare", []() { Level lev(9, 8, 7, 6); unique_ptr c1(new impl::msg::Context(lev, Trange(1, 2, 3))); unique_ptr c2(new impl::msg::Context(lev, Trange(1, 3, 2))); wassert(actual(c1->values.size()) == 0); wassert(actual(c1->level) == lev); wassert(actual(c1->trange) == Trange(1, 2, 3)); wassert(actual(c2->values.size()) == 0); wassert(actual(c2->level) == lev); wassert(actual(c2->trange) == Trange(1, 3, 2)); c1->values.set(var(WR_VAR(0, 1, 1))); c2->values.set(var(WR_VAR(0, 1, 1))); wassert(actual(c1->compare(*c2)) < 0); wassert(actual(c2->compare(*c1)) > 0); wassert(actual(c1->compare(*c1)) == 0); wassert(actual(c2->compare(*c2)) == 0); wassert(actual(c1->compare(lev, Trange(1, 2, 4))) < 0); wassert(actual(c1->compare(lev, Trange(1, 2, 2))) > 0); wassert(actual(c1->compare(lev, Trange(1, 3, 3))) < 0); wassert(actual(c1->compare(lev, Trange(1, 1, 3))) > 0); wassert(actual(c1->compare(lev, Trange(2, 2, 3))) < 0); wassert(actual(c1->compare(lev, Trange(0, 2, 3))) > 0); wassert(actual(c1->compare(Level(9, 8, 7, 7), Trange(1, 2, 3))) < 0); wassert(actual(c1->compare(Level(9, 8, 7, 5), Trange(1, 2, 3))) > 0); wassert(actual(c1->compare(lev, Trange(1, 2, 3))) == 0); }); // Test Context external ordering add_method("compare_external", []() { Trange tr(1, 2, 3); unique_ptr c1(new impl::msg::Context(Level(1, 2, 3, 4), tr)); unique_ptr c2(new impl::msg::Context(Level(2, 1, 4, 3), tr)); wassert(actual(c1->values.size()) == 0); wassert(actual(c1->level) == Level(1, 2, 3, 4)); wassert(actual(c2->values.size()) == 0); wassert(actual(c2->level) == Level(2, 1, 4, 3)); wassert(actual(c1->compare(*c2)) < 0); wassert(actual(c2->compare(*c1)) > 0); wassert(actual(c1->compare(*c1)) == 0); wassert(actual(c2->compare(*c2)) == 0); wassert(actual(c1->compare(Level(1, 2, 4, 4), tr)) < 0); wassert(actual(c1->compare(Level(1, 2, 2, 4), tr)) > 0); wassert(actual(c1->compare(Level(1, 3, 3, 4), tr)) < 0); wassert(actual(c1->compare(Level(1, 1, 3, 4), tr)) > 0); wassert(actual(c1->compare(Level(2, 2, 3, 4), tr)) < 0); wassert(actual(c1->compare(Level(0, 2, 3, 4), tr)) > 0); wassert(actual(c1->compare(Level(1, 2, 3, 4), tr)) == 0); }); // Test msg::Context internal ordering add_method("compare_internal", []() { unique_ptr c(new impl::msg::Context(Level(1, 2, 3, 4), Trange::instant())); c->values.set(var(WR_VAR(0, 1, 1))); wassert(actual(c->values.size()) == 1); c->values.set(var(WR_VAR(0, 1, 7))); wassert(actual(c->values.size()) == 2); c->values.set(var(WR_VAR(0, 1, 2))); wassert(actual(c->values.size()) == 3); // Variables with same code must get substituded and not added c->values.set(var(WR_VAR(0, 1, 1))); wassert(actual(c->values.size()) == 3); #if 0 // Check that the datum vector inside the context is in strict ascending order for (unsigned i = 0; i < c->values.size() - 1; ++i) wassert(actual_varcode(c->data[i]->code()) < c->data[i + 1]->code()); #endif wassert(actual(c->values.maybe_var(WR_VAR(0, 1, 1))).istrue()); wassert(actual_varcode(c->values.maybe_var(WR_VAR(0, 1, 1))->code()) == WR_VAR(0, 1, 1)); wassert(actual(c->values.maybe_var(WR_VAR(0, 1, 2))).istrue()); wassert(actual_varcode(c->values.maybe_var(WR_VAR(0, 1, 2))->code()) == WR_VAR(0, 1, 2)); wassert(actual(c->values.maybe_var(WR_VAR(0, 1, 7))).istrue()); wassert(actual_varcode(c->values.var(WR_VAR(0, 1, 7)).code()) == WR_VAR(0, 1, 7)); wassert(actual(c->values.maybe_var(WR_VAR(0, 1, 8))) == (const Var*)0); }); } } test("msg_context"); } dballe-8.6/dballe/msg/ltypes.txt0000644000175000017500000000517613554573614013653 000000000000000 Reserved 1 Ground or Water Surface 2 Cloud Base Level 3 Level of Cloud Tops 4 Level of 0C Isotherm 5 Level of Adiabatic Condensation Lifted from the Surface 6 Maximum Wind Level 7 Tropopause 8 Nominal Top of the Atmosphere DB-All.e encodes the channel number of polar satellites in L1 9 Sea Bottom 10-19 Reserved 20 Isothermal Level K/10 21-99 Reserved 100 Isobaric Surface Pa 101 Mean Sea Level 102 Specific Altitude Above Mean Sea Level mm 103 Specified Height Level Above Ground mm 104 Sigma Level 1/10000 105 Hybrid Level 106 Depth Below Land Surface mm 107 Isentropic (theta) Level K/10 108 Level at Specified Pressure Difference from Ground to Level Pa 109 Potential Vorticity Surface 10-9 K m2 kg-1 s-1 110 Reserved 111 Eta (NAM) Level (see note below) 1/10000 112 116 Reserved 117 Mixed Layer Depth mm 118-159 Reserved 160 Depth Below Mean Sea Level mm 161 Depth Below Water Surface mm 162-191 Reserved 200 Entire atmosphere (considered as a single layer) 201 Entire ocean (considered as a single layer) 204 Highest tropospheric freezing level 206 Grid scale cloud bottom level 207 Grid scale cloud top level 209 Boundary layer cloud bottom level 210 Boundary layer cloud top level 211 Boundary layer cloud layer 212 Low cloud bottom level 213 Low cloud top level 214 Low cloud layer 215 Cloud ceiling 220 Planetary Boundary Layer 222 Middle cloud bottom level 223 Middle cloud top level 224 Middle cloud layer 232 High cloud bottom level 233 High cloud top level 234 High cloud layer 235 Ocean Isotherm Level K/10 240 Ocean Mixed Layer 241 Ordered Sequence of Data 242 Convective cloud bottom level 243 Convective cloud top level 244 Convective cloud layer 245 Lowest level of the wet bulb zero 246 Maximum equivalent potential temperature level 247 Equilibrium level 248 Shallow convective cloud bottom level 249 Shallow convective cloud top level 251 Deep convective cloud bottom level 252 Deep convective cloud top level 253 Lowest bottom level of supercooled liquid water layer 254 Highest top level of supercooled liquid water layer 256 Clouds 257 Information about the station that generated the data 258 (use when ltype1=256) Cloud Data group, L2 = 1 low clouds, 2 middle clouds, 3 high clouds, 0 others 259 (use when ltype1=256) Individual cloud groups, L2 = group number 260 (use when ltype1=256) Cloud drift, L2 = group number 261 (use when ltype1=256) Cloud elevation, L2 = group number; (use when ltype1=264) L2 = swell wave group number 262 (use when ltype1=256) Direction and elevation of clouds, L2 is ignored 263 (use when ltype1=256) Cloud groups with bases below station level, L2 = group number 264 Waves 265 Non-physical data level dballe-8.6/dballe/msg/wr_importers/0000755000175000017500000000000013602152021014350 500000000000000dballe-8.6/dballe/msg/wr_importers/flight.cc0000644000175000017500000001640013554564112016073 00000000000000#include "base.h" #include "dballe/core/var.h" #include "dballe/msg/msg.h" #include #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { class FlightImporter : public WMOImporter { protected: Level lev; std::vector deferred; const Var* b01006; const Var* b01008; void import_var(const Var& var); public: FlightImporter(const dballe::ImporterOptions& opts) : WMOImporter(opts) {} virtual ~FlightImporter() { // If there are leftover variables in deferred, deallocate them for (std::vector::iterator i = deferred.begin(); i != deferred.end(); ++i) if (*i) delete *i; } void init() override { WMOImporter::init(); lev = Level(); deferred.clear(); b01006 = b01008 = NULL; } void acquire(const Var& var) { if (lev.ltype1 == MISSING_INT) { // If we don't have a level yet, defer adding the variable until we // have one unique_ptr copy(var_copy_without_unset_attrs(var)); deferred.push_back(copy.release()); } else msg->set(lev, Trange::instant(), var.code(), var); } void acquire(const Var& var, Varcode code) { if (lev.ltype1 == MISSING_INT) { // If we don't have a level yet, defer adding the variable until we // have one unique_ptr copy(var_copy_without_unset_attrs(var, code)); deferred.push_back(copy.release()); } else msg->set(lev, Trange::instant(), code, var); } void set_level(const Level& newlev) { if (lev.ltype1 != MISSING_INT) error_consistency::throwf("found two flight levels: %s and %s", lev.describe().c_str(), newlev.describe().c_str()); lev = newlev; // Flush deferred variables for (vector::iterator i = deferred.begin(); i != deferred.end(); ++i) { unique_ptr var(*i); *i = 0; msg->set(lev, Trange::instant(), move(var)); } deferred.clear(); } void run() override { for (pos = 0; pos < subset->size(); ++pos) { const Var& var = (*subset)[pos]; if (WR_VAR_F(var.code()) != 0) continue; if (var.isset()) import_var(var); } if (b01008) { msg->set_ident_var(*b01008); if (b01006) acquire(*b01006); } else if (b01006) msg->set_ident_var(*b01006); } MessageType scanTypeFromVars(const Subset& subset) const { for (unsigned i = 0; i < subset.size(); ++i) { switch (subset[i].code()) { case WR_VAR(0, 2, 65): // ACARS GROUND RECEIVING STATION if (subset[0].isset()) return MessageType::ACARS; break; } } return MessageType::AMDAR; } MessageType scanType(const Bulletin& bulletin) const { switch (bulletin.data_subcategory_local) { case 142: return MessageType::AIREP; case 144: return MessageType::AMDAR; case 145: return MessageType::ACARS; default: // Scan for the presence of significant B codes if (bulletin.subsets.empty()) return MessageType::GENERIC; return scanTypeFromVars(bulletin.subsets[0]); } } }; std::unique_ptr Importer::createFlight(const dballe::ImporterOptions& opts) { return unique_ptr(new FlightImporter(opts)); } void FlightImporter::import_var(const Var& var) { switch (var.code()) { case WR_VAR(0, 1, 6): b01006 = &var; break; case WR_VAR(0, 1, 8): b01008 = &var; break; case WR_VAR(0, 1, 23): acquire(var); break; case WR_VAR(0, 2, 1): acquire(var); break; case WR_VAR(0, 2, 2): acquire(var); break; case WR_VAR(0, 2, 5): acquire(var); break; case WR_VAR(0, 2, 61): acquire(var); break; case WR_VAR(0, 2, 62): acquire(var); break; case WR_VAR(0, 2, 63): acquire(var); break; case WR_VAR(0, 2, 64): acquire(var); break; case WR_VAR(0, 2, 70): acquire(var); break; case WR_VAR(0, 7, 2): // Specific Altitude Above Mean Sea Level in mm set_level(Level(102, var.enqd() * 1000)); acquire(var, WR_VAR(0, 7, 30)); break; case WR_VAR(0, 7, 4): // Isobaric Surface in Pa if (lev.ltype1 == MISSING_INT) set_level(Level(100, var.enqd())); acquire(var, WR_VAR(0, 10, 4)); break; case WR_VAR(0, 7, 10): // Flight level if (opts.simplified) { // Convert to pressure using formula from // http://www.wmo.int/pages/prog/www/IMOP/publications/CIMO-Guide/CIMO%20Guide%207th%20Edition,%202008/Part%20II/Chapter%203.pdf double p_hPa = 1013.25 * pow(1.0 - 0.000001 * 6.8756 * var.enqd() * 3.28084, 5.2559); set_level(Level(100, round(p_hPa * 100))); } else // Specific Altitude Above Mean Sea Level in mm set_level(Level(102, var.enqd() * 1000)); acquire(var, WR_VAR(0, 7, 30)); break; case WR_VAR(0, 8, 4): acquire(var); break; case WR_VAR(0, 8, 9): acquire(var); break; case WR_VAR(0, 8, 21): acquire(var); break; case WR_VAR(0, 11, 1): acquire(var); break; case WR_VAR(0, 11, 2): acquire(var); break; case WR_VAR(0, 11, 31): acquire(var); break; case WR_VAR(0, 11, 32): acquire(var); break; case WR_VAR(0, 11, 33): acquire(var); break; case WR_VAR(0, 11, 34): acquire(var); break; case WR_VAR(0, 11, 35): acquire(var); break; case WR_VAR(0, 11, 36): acquire(var); break; case WR_VAR(0, 11, 37): acquire(var); break; case WR_VAR(0, 11, 39): acquire(var); break; case WR_VAR(0, 11, 77): acquire(var); break; case WR_VAR(0, 12, 1): acquire(var, WR_VAR(0, 12, 101)); break; case WR_VAR(0, 12,101): acquire(var); break; case WR_VAR(0, 12, 3): acquire(var, WR_VAR(0, 12, 103)); break; case WR_VAR(0, 12,103): acquire(var); break; case WR_VAR(0, 13, 2): acquire(var); break; case WR_VAR(0, 13, 3): acquire(var); break; case WR_VAR(0, 20, 41): acquire(var); break; case WR_VAR(0, 20, 42): acquire(var); break; case WR_VAR(0, 20, 43): acquire(var); break; case WR_VAR(0, 20, 44): acquire(var); break; case WR_VAR(0, 20, 45): acquire(var); break; case WR_VAR(0, 33, 25): acquire(var); break; // TODO: repeated 011075 MEAN TURBULENCE INTENSITY (EDDY DISSIPATION RATE)[M**(2/3)/S] // TODO: repeated 011076 PEAK TURBULENCE INTENSITY (EDDY DISSIPATION RATE)[M**(2/3)/S] default: WMOImporter::import_var(var); break; } } } } } } dballe-8.6/dballe/msg/wr_importers/generic.cc0000644000175000017500000001161113554564112016231 00000000000000#include "base.h" #include "dballe/core/var.h" #include "dballe/msg/msg.h" #include #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { class GenericImporter : public Importer { protected: Level lev; Trange tr; /// Import an undefined value void import_defined(const Var& var); /// Import a defined value, which can be a variable or context void import_undef(const Var& var); /// Import a defined value, with the context being properly set void import_var(const Var& var); public: GenericImporter(const dballe::ImporterOptions& opts) : Importer(opts) {} virtual ~GenericImporter() {} void init() override { Importer::init(); lev = Level(); tr = Trange(); } void run() override { for (size_t pos = 0; pos < subset->size(); ++pos) { const Var& var = (*subset)[pos]; // Skip non-variable entries if (WR_VAR_F(var.code()) != 0) continue; // Special processing for undefined variables if (!var.isset()) { // Also skip attributes of undefined variables if there are // some following for ( ; pos + 1 < subset->size() && WR_VAR_X((*subset)[pos + 1].code()) == 33; ++pos) ; import_undef(var); continue; } // A variable with a value: add attributes to it if any are // found if (pos + 1 < subset->size() && WR_VAR_X((*subset)[pos + 1].code()) == 33) { Var copy(var); for ( ; pos + 1 < subset->size() && WR_VAR_X((*subset)[pos + 1].code()) == 33; ++pos) copy.seta((*subset)[pos + 1]); import_defined(copy); } else import_defined(var); } } MessageType scanType(const Bulletin&) const override { return MessageType::GENERIC; } }; std::unique_ptr Importer::createGeneric(const dballe::ImporterOptions& opts) { return unique_ptr(new GenericImporter(opts)); } void GenericImporter::import_undef(const Var& var) { switch (var.code()) { case WR_VAR(0, 4, 192): tr.pind = MISSING_INT; break; case WR_VAR(0, 4, 193): tr.p1 = MISSING_INT; break; case WR_VAR(0, 4, 194): tr.p2 = MISSING_INT; break; case WR_VAR(0, 7, 192): lev.ltype1 = MISSING_INT; break; case WR_VAR(0, 7, 193): lev.l1 = MISSING_INT; break; case WR_VAR(0, 7, 194): lev.l2 = MISSING_INT; break; case WR_VAR(0, 7, 195): lev.ltype2 = MISSING_INT; break; } } void GenericImporter::import_defined(const Var& var) { switch (var.code()) { case WR_VAR(0, 4, 192): tr.pind = var.enqi(); break; case WR_VAR(0, 4, 193): tr.p1 = var.enqi(); break; case WR_VAR(0, 4, 194): tr.p2 = var.enqi(); break; case WR_VAR(0, 7, 192): lev.ltype1 = var.enqi(); break; case WR_VAR(0, 7, 193): lev.l1 = var.enqi(); break; case WR_VAR(0, 7, 194): lev.l2 = var.enqi(); break; case WR_VAR(0, 7, 195): lev.ltype2 = var.enqi(); break; case WR_VAR(0, 1, 194): if (var.isset()) { // Set the rep memo if we found it const char* repmemo = var.enqc(); msg->type = Message::type_from_repmemo(repmemo); msg->set_rep_memo(repmemo, -1); } break; default: import_var(var); break; } } void GenericImporter::import_var(const Var& var) { // Adjust station info level for pre-dballe-5.0 generics if (lev.ltype1 == 257) { lev = Level(); tr = Trange(); } switch (var.code()) { // Legacy variable conversions case WR_VAR(0, 8, 1): { unique_ptr nvar(newvar(WR_VAR(0, 8, 42), (int)convert_BUFR08001_to_BUFR08042(var.enqi()))); nvar->setattrs(var); msg->set(lev, tr, move(nvar)); break; } // Datetime entries that may have attributes to store case WR_VAR(0, 4, 1): msg->set_year_var(var); break; case WR_VAR(0, 4, 2): msg->set_month_var(var); break; case WR_VAR(0, 4, 3): msg->set_day_var(var); break; case WR_VAR(0, 4, 4): msg->set_hour_var(var); break; case WR_VAR(0, 4, 5): msg->set_minute_var(var); break; case WR_VAR(0, 4, 6): msg->set_second_var(var); break; // Anything else default: msg->set(lev, tr, map_code_to_dballe(var.code()), var); break; } } } } } } dballe-8.6/dballe/msg/wr_importers/pollution.cc0000644000175000017500000003135613554564112016652 00000000000000#include "base.h" #include "dballe/msg/msg.h" #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { static double intexp10(unsigned x) { switch (x) { case 0: return 1.0; case 1: return 10.0; case 2: return 100.0; case 3: return 1000.0; case 4: return 10000.0; case 5: return 100000.0; case 6: return 1000000.0; case 7: return 10000000.0; case 8: return 100000000.0; case 9: return 1000000000.0; case 10: return 10000000000.0; case 11: return 100000000000.0; case 12: return 1000000000000.0; case 13: return 10000000000000.0; case 14: return 100000000000000.0; case 15: return 1000000000000000.0; case 16: return 10000000000000000.0; default: error_domain::throwf("computing double value of %u^10 is not yet supported", x); } } #define MISSING_PRESS -1.0 static inline int to_h(double val) { return lround(val / 9.80665); } class PollutionImporter : public WMOImporter { protected: Level lev; Trange tr; int valtype; int decscale; int value; const Var* attr_conf; const Var* attr_cas; const Var* attr_pmc; const Var* finalvar; void import_var(const Var& var); public: PollutionImporter(const dballe::ImporterOptions& opts) : WMOImporter(opts) {} virtual ~PollutionImporter() {} virtual void init() { WMOImporter::init(); lev = Level(103); tr = Trange(0); valtype = 0; decscale = MISSING_INT; value = 0; attr_conf = NULL; attr_cas = NULL; attr_pmc = NULL; finalvar = NULL; } virtual void run() { // Scan the input variables for (pos = 0; pos < subset->size(); ++pos) { const Var& var = (*subset)[pos]; if (WR_VAR_F(var.code()) != 0) continue; if (var.isset()) import_var(var); } // Create the final pollutant variables by putting all the pieces // together // Use default level and time range if the message did not report it if (lev.l1 == MISSING_INT) lev.l1 = 3000; if (tr.p1 == MISSING_INT) { tr.p1 = -3600; tr.p2 = 3600; } // Create the final variable unique_ptr finalvar = newvar(valtype); // Scale the value and set it if (decscale > 0) finalvar->setd(value * intexp10(decscale)); else finalvar->setd(value / intexp10(-decscale)); // Add the attributes if (attr_conf) finalvar->seta(*attr_conf); if (attr_cas) finalvar->seta(*attr_cas); if (attr_pmc) finalvar->seta(*attr_pmc); // Store it into the dba_msg msg->set(lev, tr, move(finalvar)); } MessageType scanType(const Bulletin& bulletin) const { return MessageType::POLLUTION; } }; std::unique_ptr Importer::createPollution(const dballe::ImporterOptions& opts) { return unique_ptr(new PollutionImporter(opts)); } void PollutionImporter::import_var(const Var& var) { switch (var.code()) { /* For this parameter you can give up to 32 characters as a station * name. */ case WR_VAR(0, 1, 19): msg->set_st_name_var(var); break; /* Airbase local code -- Up to 7 characters reflecting the local * station code supplied with the observations. If not given then * leave blank. */ case WR_VAR(0, 1, 212): msg->set_poll_lcode_var(var); break; /* Airbase station code -- 7 character code supplied with AirBase * observations (see Ref 1, II.1.4, page 23). If not supplied then * leave blank.*/ case WR_VAR(0, 1, 213): msg->set_poll_scode_var(var); break; /* GEMS code -- 6 character code suggested at RAQ Paris meeting, * December 2006. First 2 characters to be country code (using * ISO 3166-1-alpha-2 code), next 4 characters to be unique station * number within national boundary for each station (numbering to * be defined and maintained by each GEMS RAQ partner responsible * for collecting observations within each national boundar * invovled)*/ case WR_VAR(0, 1, 214): msg->set_poll_gemscode_var(var); break; /* * Dominant emission source influencing the air pollution * concentrations at the station (based on Ref 1, II.2.2, page 28) * Possible values are: * 0 traffic * 1 industrial * 2 background * 3-6 reserved (do not use) * 7 missing (or unknown) */ case WR_VAR(0, 1, 215): msg->set_poll_source_var(var); break; /* * Type of area in which station is located (based on Ref 1, II.2.1, page 27) * Possible values are: * 0 urban * 1 suburban * 2 rural * 3-6 reserved (do not use) * 7 missing (or unknown) */ case WR_VAR(0, 1, 216): msg->set_poll_atype_var(var); break; /* * Type of terrain in which the station is located (based on table in Ref 1, II.1.12, page 26) * Possible values are: * 0 mountain * 1 valley * 2 seaside * 3 lakeside * 4 plain * 5 hilly terrain * 6-14 reserved (do not use) * 15 missing (or unknown) */ case WR_VAR(0, 1, 217): msg->set_poll_ttype_var(var); break; /* * Height of station above mean sea level and height of sensing * instrument above local ground. Both in metres. If not known than * can be coded as missing value. */ case WR_VAR(0, 7, 30): msg->set_height_station_var(var); break; case WR_VAR(0, 7, 31): lev.l1 = var.enqi(); break; /* Signifies that observation is an average over a certain time period. Value set to 2. */ case WR_VAR(0, 8, 21): if (var.enqi() != 2) error_consistency::throwf("time significance is %d instead of 2", var.enqi()); break; /* * Time period over which the average has been taken (in minutes), * e.g. -60 for average over the previous hour. The period is * relative to the date/time of the observation. */ case WR_VAR(0, 4, 25): // Convert from minutes to seconds tr.p1 = var.enqi() * 60; tr.p2 = -tr.p1; break; /* * VAL stands for validation and signifies that this parameter has * not yet reached operational status at WMO. * Identifier for species observed. * Possible values are: * Code Constituent CAS Registry Number (if applicable) * 0 Ozone (O3) 10028-15-6 * 1 Water vapour (H2O) 7732-18-5 * 2 Methane (CH4) 74-82-8 * 3 Carbon dioxide (CO2) 37210-16-5 * 4 Carbon monoxide (CO) 630-08-0 * 5 Nitrogen dioxide (NO2) 10102-44-0 * 6 Nitrous oxide (N2O) 10024-97-2 * 7 Formaldehyde (HCHO) 50-00-0 * 8 Sulphur dioxide (SO2) 7446-09-5 * 9-24 reserved * 25 Particulate matter < 1.0 microns * 26 Particulate matter < 2.5 microns * 27 Particulate matter < 10 microns * 28 Aerosols (generic) * 29 Smoke (generic) * 30 Crustal material (generic) * 31 Volcanic ash * 32-200 reserved * 201-254 reserved for local use * 255 missing * We may have to propose some new entries to WMO if this does not * cover the range of constituents of the air quality observations. * N.B. Do not code this as missing. This is a key piece of * information in identifying the observed quantity. * * Our target BUFR entries are: * 015192 [SIM] NO Concentration Does not fit in above table: not exported * 015193 [SIM] NO2 Concentration 5 * 015194 [SIM] O3 Concentration 0 * 015195 [SIM] PM10 Concentration 27 */ case WR_VAR(0, 8, 43): switch (var.enqi()) { case 0: valtype = WR_VAR(0, 15, 194); break; case 4: valtype = WR_VAR(0, 15, 196); break; case 5: valtype = WR_VAR(0, 15, 193); break; case 8: valtype = WR_VAR(0, 15, 197); break; case 26: valtype = WR_VAR(0, 15, 198); break; case 27: valtype = WR_VAR(0, 15, 195); break; default: error_consistency::throwf("cannot import constituent %d as there is no mapping for it", var.enqi()); } break; /* * Chemical Abstracts Service (CAS) Registry number of constituent, * if applicable. This parameter is optional and can be coded as a * blank character string. */ case WR_VAR(0, 8, 44): attr_cas = &var; break; /* * If parameter 008043 is coded as 25, 26 or 27, then this * parameter can be used to further categorise the nature of the * particulate matter. * * Possible values are: * 0 Particulate matter (all types) * 1 NO3(-) * 2 NH4(+) * 3 Na(+) * 4 Cl(-) * 5 Ca(2+) * 6 Mg(2+) * 7 K(+) * 8 SO4(2-) * 9-200 reserved * 201-254 reserved for local use * 255 missing */ case WR_VAR(0, 8, 45): attr_pmc = &var; break; /* * A recent feature (still pre-operational at WMO) is the * introduction of scaled quanities specifically to deal with * quantities which may exhibit a large dynamic range. In order to * be able to cover a large dynamic range whilst conserving * precision some specific scale quantities have been introduced * which have an associated decimal scaling factor. The * example_airbase2bufr.f90 code gives an example of a method to * calculate the decimal scaling factor. */ case WR_VAR(0, 8, 90): /* Someone seems to have thought that C fields in BUFR data * section were not crazy enough, and went on reimplementing * them using B fields. So we have to reimplement the same * logic here. * * I'll however ignore resetting the decimal scale because in * this template it is only used for the measured pollutant * value. */ if (decscale == MISSING_INT) { decscale = var.enqi(); /* * Sadly, someone seems to have decided that "all 8 bits to * 1" missing data is the same as encoding -127, so they * encode "all 8 bits to 0 with a reference value of -127" */ if (decscale == -127) decscale = 0; } break; /* * This is the most suitable parameter (that I can find) in BUFR to * represent the concentration of pollutants. The units are * kg/m**3. The example_airbase2bufr.f90 code gives an example of a * method to calculate the decimal scaling factor and scaled mass * density from the observed concentration. To do the backwards * calculation, concentration (kg/m**3) = scaled mass density * * 10**(decimal scaling factor) */ case WR_VAR(0, 15, 23): value = var.enqi(); break; /* * Parameter to give a qualitative measure of the quality of the * observation. Set at the discretion of the encoder given any * information they have either directly from the observation data * set or otherwise. * Possible values are: * 0 Data not suspect * 1 Data slightly suspect * 2 Data highly suspect * 3 Data considered unfit for use * 4-6 Reserved * 7 Quality information not given */ case WR_VAR(0, 33, 3): attr_conf = &var; break; default: WMOImporter::import_var(var); break; } } } } } } dballe-8.6/dballe/msg/wr_importers/metar.cc0000644000175000017500000000617413554564112015735 00000000000000#include "base.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/msg.h" #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { #define MISSING_SENSOR_H -10000 class MetarImporter : public WMOImporter { protected: double height_sensor; void peek_var(const Var& var); void import_var(const Var& var); void set_gen_sensor(const Var& var, Varcode code, const Level& defaultLevel, const Trange& trange) { if (height_sensor == MISSING_SENSOR_H || defaultLevel == Level(103, height_sensor * 1000)) msg->set(defaultLevel, trange, code, var); else if (opts.simplified) { Var var1(var); var1.seta(newvar(WR_VAR(0, 7, 32), height_sensor)); msg->set(defaultLevel, trange, code, var1); } else msg->set(Level(103, height_sensor * 1000), trange, code, var); } void set_gen_sensor(const Var& var, const Shortcut& shortcut) { set_gen_sensor(var, shortcut.code, shortcut.level, shortcut.trange); } public: MetarImporter(const dballe::ImporterOptions& opts) : WMOImporter(opts) {} virtual ~MetarImporter() {} virtual void init() { WMOImporter::init(); height_sensor = MISSING_SENSOR_H; } virtual void run() { for (pos = 0; pos < subset->size(); ++pos) { const Var& var = (*subset)[pos]; if (WR_VAR_F(var.code()) != 0) continue; if (WR_VAR_X(var.code()) < 10) peek_var(var); if (var.isset()) import_var(var); } } MessageType scanType(const Bulletin& bulletin) const { return MessageType::METAR; } }; std::unique_ptr Importer::createMetar(const dballe::ImporterOptions& opts) { return unique_ptr(new MetarImporter(opts)); } void MetarImporter::peek_var(const Var& var) { switch (var.code()) { // Context items case WR_VAR(0, 7, 6): if (var.isset()) height_sensor = var.enqi(); else height_sensor = MISSING_SENSOR_H; break; } } void MetarImporter::import_var(const Var& var) { switch (var.code()) { case WR_VAR(0, 7, 1): msg->set_height_station_var(var); break; case WR_VAR(0, 11, 1): set_gen_sensor(var, sc::wind_dir); break; case WR_VAR(0, 11, 16): set_gen_sensor(var, sc::ex_ccw_wind); break; case WR_VAR(0, 11, 17): set_gen_sensor(var, sc::ex_cw_wind); break; case WR_VAR(0, 11, 2): set_gen_sensor(var, sc::wind_speed); break; case WR_VAR(0, 11, 41): set_gen_sensor(var, sc::wind_speed); break; case WR_VAR(0, 12, 1): set_gen_sensor(var, sc::temp_2m); break; case WR_VAR(0, 12, 3): set_gen_sensor(var, sc::dewpoint_2m); break; case WR_VAR(0, 10, 52): set_gen_sensor(var, sc::qnh); break; case WR_VAR(0, 20, 9): set_gen_sensor(var, sc::metar_wtr); break; default: WMOImporter::import_var(var); break; } } } } } } dballe-8.6/dballe/msg/wr_importers/base.cc0000644000175000017500000005317613554564112015543 00000000000000#include "base.h" #include "dballe/core/var.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/msg.h" #include #include #include #include #include #define MISSING_BARO -10000.0 #define MISSING_PRESS_STD 0.0 #define MISSING_TIME_SIG -10000 using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { static const Trange tr_std_past_wtr3(205, 0, 10800); static const Trange tr_std_past_wtr6(205, 0, 21600); static const Level lev_std_wind(103, 10*1000); static const Trange tr_std_wind_max10m(205, 0, 600); void Importer::init() { } void Importer::import(const wreport::Subset& subset, Message& msg) { this->subset = ⊂ this->msg = &msg; init(); run(); } void Importer::set(const wreport::Var& var, const Shortcut& shortcut) { msg->set(shortcut, var); } void Importer::set(const wreport::Var& var, wreport::Varcode code, const Level& level, const Trange& trange) { msg->set(level, trange, code, var); } std::unique_ptr Importer::createSat(const dballe::ImporterOptions&) { throw error_unimplemented("WB sat Importers"); } void WMOImporter::import_var(const Var& var) { switch (var.code()) { // General bulletin metadata case WR_VAR(0, 1, 1): set(var, sc::block); break; case WR_VAR(0, 1, 2): set(var, sc::station); break; case WR_VAR(0, 1, 5): case WR_VAR(0, 1, 6): case WR_VAR(0, 1, 11): set(var, sc::ident); break; case WR_VAR(0, 1, 12): set(var, sc::st_dir); break; case WR_VAR(0, 1, 13): set(var, sc::st_speed); break; case WR_VAR(0, 1, 63): set(var, sc::st_name_icao); break; case WR_VAR(0, 2, 1): set(var, sc::st_type); break; case WR_VAR(0, 1, 15): set(var, sc::st_name); break; case WR_VAR(0, 4, 1): set(var, sc::year); break; case WR_VAR(0, 4, 2): set(var, sc::month); break; case WR_VAR(0, 4, 3): set(var, sc::day); break; case WR_VAR(0, 4, 4): set(var, sc::hour); break; case WR_VAR(0, 4, 5): set(var, sc::minute); break; case WR_VAR(0, 4, 6): set(var, sc::second); break; case WR_VAR(0, 5, 1): case WR_VAR(0, 5, 2): set(var, sc::latitude); break; case WR_VAR(0, 6, 1): case WR_VAR(0, 6, 2): set(var, sc::longitude); break; } } void LevelContext::init() { height_baro = MISSING_BARO; press_std = MISSING_PRESS_STD; height_sensor = missing; height_sensor_seen = false; sea_depth = missing; ground_depth = missing; swell_wave_group = false; } void LevelContext::peek_var(const wreport::Var& var) { switch (var.code()) { case WR_VAR(0, 7, 4): // Remember the standard level pressure to use later as layer for geopotential press_std = var.enq(MISSING_PRESS_STD); break; case WR_VAR(0, 7, 31): // Remember the height to use later as layer for pressure height_baro = var.enq(MISSING_BARO); break; case WR_VAR(0, 7, 32): // Height to use later as level for whatever needs it height_sensor = missing; height_sensor = var.enq(missing); height_sensor_seen = true; break; case WR_VAR(0, 7, 61): ground_depth = var.enq(missing); break; case WR_VAR(0, 7, 63): sea_depth = var.enq(missing); break; case WR_VAR(0, 22, 3): swell_wave_group=true; break; } } void TimerangeContext::init() { time_period = MISSING_INT; time_period_offset = 0; time_period_seen = false; time_sig = MISSING_TIME_SIG; hour = MISSING_INT; last_B04024_pos = -1; } void TimerangeContext::peek_var(const Var& var, unsigned pos) { if (var.isset()) { switch (var.code()) { case WR_VAR(0, 4, 4): hour = var.enqi(); break; case WR_VAR(0, 4, 24): // Time period in hours if ((int)pos == last_B04024_pos + 1) { // Cope with the weird idea of using B04024 twice to indicate // beginning and end of a period not ending with the SYNOP // reference time if (time_period != MISSING_INT && var.enqi() != 0) { time_period -= var.enqd() * 3600; time_period_offset = var.enqd() * 3600; } } else { time_period = var.enqd() * 3600; time_period_seen = true; time_period_offset = 0; } last_B04024_pos = pos; break; case WR_VAR(0, 4, 25): // Time period in minutes time_period = var.enqd() * 60; time_period_seen = true; time_period_offset = 0; break; case WR_VAR(0, 8, 21): // Time significance time_sig = var.enqi(); // If we get time significance 18 "Radiosonde launch time" // before getting the initial reference time, they are trying // to tell us that they are giving us the radiosonde launch // time: ignore it, since it is already what we gave for // granted. if (hour == MISSING_INT and time_sig == 18) time_sig = MISSING_TIME_SIG; break; } } else { switch (var.code()) { case WR_VAR(0, 4, 4): hour = MISSING_INT; break; case WR_VAR(0, 4, 24): // Time period in hours time_period = MISSING_INT; time_period_offset = 0; time_period_seen = true; break; case WR_VAR(0, 4, 25): // Time period in minutes time_period = MISSING_INT; time_period_offset = 0; time_period_seen = true; break; case WR_VAR(0, 8, 21): // Time significance time_sig = MISSING_TIME_SIG; break; } } } void CloudContext::init() { level = Level::cloud(MISSING_INT, MISSING_INT); } void CloudContext::on_vss(const wreport::Subset& subset, unsigned pos) { /* Vertical significance */ if (pos == 0) throw error_consistency("B08002 found at beginning of message"); Varcode prev = subset[pos - 1].code(); if (prev == WR_VAR(0, 20, 10)) { // Normal cloud data level.ltype2 = 258; level.l2 = 0; return; } if (pos == subset.size() - 1) throw error_consistency("B08002 found at end of message"); Varcode next = subset[pos + 1].code(); switch (next) { case WR_VAR(0, 20, 11): { if (pos >= subset.size() - 3) throw error_consistency("B08002 followed by B20011 found less than 3 places before end of message"); Varcode next2 = subset[pos + 3].code(); if (next2 == WR_VAR(0, 20, 14)) { // Clouds with bases below station level if (level.ltype2 != 263) { level.ltype2 = 263; level.l2 = 1; } else { ++level.l2; } } else { /* Individual cloud groups */ if (level.ltype2 != 259) { level.ltype2 = 259; level.l2 = 1; } else { ++level.l2; } } break; } case WR_VAR(0, 20, 54): // Direction of cloud drift if (level.ltype2 != 260) { level.ltype2 = 260; level.l2 = 1; } else { ++level.l2; } break; default: break; #if 0 /* Vertical significance */ if (pos == 0) throw error_consistency("B08002 found at beginning of message"); if (pos == subset->size() - 1) throw error_consistency("B08002 found at end of message"); Varcode prev = (*subset)[pos - 1].code(); Varcode next = (*subset)[pos + 1].code(); } else if (var.value() == NULL) { level.ltype2 = 0; } else { /* Unless we can detect known buggy situations, raise an error */ if (next != WR_VAR(0, 20, 62)) error_consistency::throwf("Vertical significance %d found in unrecognised context", var.enqi()); } break; #endif } } const Level& CloudContext::clcmch() { if (level.ltype2 == 258) ++level.l2; return level; } void UnsupportedContext::init() { B08023 = nullptr; } bool UnsupportedContext::is_unsupported() const { return B08023 != nullptr; } void UnsupportedContext::peek_var(const wreport::Var& var, unsigned pos) { switch (var.code()) { case WR_VAR(0, 8, 23): if (var.isset()) B08023 = &var; else B08023 = nullptr; break; } } Interpreted::Interpreted(const Shortcut& shortcut, const wreport::Var& var) { level = shortcut.level; trange = shortcut.trange; this->var = var_copy_without_unset_attrs(var, shortcut.code); } Interpreted::Interpreted(const Shortcut& shortcut, const wreport::Var& var, const Level& level, const Trange& trange) : level(level), trange(trange) { this->var = var_copy_without_unset_attrs(var, shortcut.code); } Interpreted::Interpreted(wreport::Varcode code, const wreport::Var& var, const Level& level, const Trange& trange) : level(level), trange(trange) { this->var = var_copy_without_unset_attrs(var, code); } Interpreted::~Interpreted() { } void InterpretedPrecise::set_sensor_height(const LevelContext& ctx) { if (ctx.height_sensor == 0) level = Level(1); else if (ctx.height_sensor_seen) level = ctx.height_sensor == LevelContext::missing ? Level(103) : Level(103, round(ctx.height_sensor * 1000.0)); } void InterpretedSimplified::set_sensor_height(const LevelContext& ctx) { if (ctx.height_sensor == 0) return; if (!ctx.height_sensor_seen) return; if (ctx.height_sensor == LevelContext::missing) return; if (level == Level(103, ctx.height_sensor * 1000)) return; var->seta(newvar(WR_VAR(0, 7, 32), ctx.height_sensor)); if (level.ltype1 == 103 && level.l1 != MISSING_INT) level_deviation = abs(level.l1 - (int)(ctx.height_sensor * 1000)); } void InterpretedPrecise::set_barometer_height(const LevelContext& ctx) { if (ctx.height_baro == MISSING_BARO) return; level = Level(102, ctx.height_baro * 1000); } void InterpretedSimplified::set_barometer_height(const LevelContext& ctx) { if (ctx.height_baro == MISSING_BARO) return; var->seta(newvar(WR_VAR(0, 7, 31), ctx.height_baro)); if (level.ltype1 == 102 && level.l1 != MISSING_INT) level_deviation = abs(level.l1 - (int)(ctx.height_baro * 1000)); } void InterpretedPrecise::set_duration(const TimerangeContext& ctx) { if (trange.pind == 254) trange = Trange::instant(); else if (ctx.time_period_seen) trange = ctx.time_period == MISSING_INT ? Trange(trange.pind, 0) : Trange(trange.pind, 0, abs(ctx.time_period)); } void InterpretedSimplified::set_duration(const TimerangeContext& ctx) { if (trange.pind == 254) return; if (!ctx.time_period_seen) return; if (ctx.time_period == MISSING_INT) return; Trange real = Trange(trange.pind, 0, abs(ctx.time_period)); if (trange == real) return; var->seta(newvar(WR_VAR(0, 4, 194), abs(ctx.time_period))); } void InterpretedPrecise::set_wind_mean(const TimerangeContext& ctx) { if (!ctx.time_period_seen) trange = Trange(200, 0, 600); else trange = ctx.time_period == MISSING_INT ? Trange(200, 0) : Trange(200, 0, abs(ctx.time_period)); } void InterpretedSimplified::set_wind_mean(const TimerangeContext& ctx) { if (!ctx.time_period_seen) return; if (ctx.time_period == MISSING_INT) return; Trange real = Trange(200, 0, abs(ctx.time_period)); if (real.p2 == 600) return; if (real == trange) return; var->seta(newvar(WR_VAR(0, 4, 194), abs(ctx.time_period))); } void SynopBaseImporter::set_gen_sensor(const Var& var, Varcode code, const Level& lev_std, const Trange& tr_std) { if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, code, var, lev_std, tr_std); res->set_sensor_height(level); res->set_duration(trange); set(move(res)); } void SynopBaseImporter::set_gen_sensor(const Var& var, const Shortcut& shortcut) { if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, shortcut, var); res->set_sensor_height(level); res->set_duration(trange); set(move(res)); } void SynopBaseImporter::set_baro_sensor(const Var& var, const Shortcut& shortcut) { if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, shortcut, var); res->set_barometer_height(level); res->set_duration(trange); set(move(res)); } void SynopBaseImporter::set_past_weather(const wreport::Var& var, const Shortcut& shortcut) { if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, shortcut, var); res->trange = Trange((trange.hour % 6 == 0) ? tr_std_past_wtr6 : tr_std_past_wtr3); res->set_duration(trange); set(move(res)); } void SynopBaseImporter::set_wind(const wreport::Var& var, const Shortcut& shortcut) { if (trange.time_sig != MISSING_TIME_SIG && trange.time_sig != 2) error_consistency::throwf("Found unsupported time significance %d for wind direction", trange.time_sig); if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, shortcut, var); res->level = lev_std_wind; res->set_sensor_height(level); res->set_wind_mean(trange); set(move(res)); } void SynopBaseImporter::set_wind_max(const wreport::Var& var, const Shortcut& shortcut) { if (unsupported.is_unsupported()) return; auto res = create_interpreted(opts.simplified, shortcut, var, lev_std_wind, tr_std_wind_max10m); res->set_sensor_height(level); // Always use real trange if (trange.time_period_seen) res->trange = trange.time_period == MISSING_INT ? Trange(205, 0) : Trange(205, 0, abs(trange.time_period)); set(move(res)); } void SynopBaseImporter::set_pressure(const wreport::Var& var) { if (level.press_std == MISSING_PRESS_STD) set(var, WR_VAR(0, 10, 8), Level(100), Trange::instant()); else set(var, WR_VAR(0, 10, 8), Level(100, level.press_std), Trange::instant()); } void SynopBaseImporter::set(const wreport::Var& var, const Shortcut& shortcut) { if (unsupported.is_unsupported()) return; WMOImporter::set(var, shortcut); } void SynopBaseImporter::set(const wreport::Var& var, wreport::Varcode code, const Level& level, const Trange& trange) { if (unsupported.is_unsupported()) return; WMOImporter::set(var, code, level, trange); } void SynopBaseImporter::set(std::unique_ptr val) { if (opts.simplified) queued.push_back(val.release()); else msg->set(val->level, val->trange, move(val->var)); } SynopBaseImporter::SynopBaseImporter(const dballe::ImporterOptions& opts) : WMOImporter(opts) { } SynopBaseImporter::~SynopBaseImporter() { for (auto& i: queued) delete i; } void SynopBaseImporter::init() { WMOImporter::init(); clouds.init(); level.init(); trange.init(); unsupported.init(); for (auto& i: queued) delete i; queued.clear(); } void SynopBaseImporter::run() { for (pos = 0; pos < subset->size(); ++pos) { const Var& var = (*subset)[pos]; if (WR_VAR_F(var.code()) != 0) continue; if (WR_VAR_X(var.code()) < 10 || var.code() == WR_VAR(0, 22, 3)) peek_var(var); if (var.isset()) import_var(var); } std::sort(queued.begin(), queued.end(), [](const Interpreted* a, const Interpreted* b) { if (a->level < b->level) return true; if (a->level > b->level) return false; if (a->trange < b->trange) return true; if (a->trange > b->trange) return false; if (a->level_deviation > b->level_deviation) return true; return false; }); for (auto& i: queued) { msg->set(i->level, i->trange, move(i->var)); delete i; i = nullptr; } queued.clear(); } void SynopBaseImporter::peek_var(const Var& var) { unsupported.peek_var(var, pos); level.peek_var(var); switch (var.code()) { case WR_VAR(0, 4, 4): case WR_VAR(0, 4, 24): case WR_VAR(0, 4, 25): case WR_VAR(0, 8, 21): trange.peek_var(var, pos); break; case WR_VAR(0, 8, 2): clouds.on_vss(*subset, pos); break; } } void SynopBaseImporter::import_var(const Var& var) { switch (var.code()) { case WR_VAR(0, 8, 2): // Store original VS value as a measured value set(var, WR_VAR(0, 8, 2), clouds.level, Trange::instant()); break; // Ship identification, movement, date/time, horizontal and vertical // coordinates case WR_VAR(0, 7, 1): case WR_VAR(0, 7, 30): set(var, sc::height_station); break; case WR_VAR(0, 7, 31): /* Store also in the ana level, so that if the * pressure later is missing we still have * access to the value */ set(var, sc::height_baro); break; // Pressure data (complete) case WR_VAR(0, 10, 4): set_baro_sensor(var, sc::press); break; case WR_VAR(0, 10, 51): set_baro_sensor(var, sc::press_msl); break; case WR_VAR(0, 10, 61): set_baro_sensor(var, sc::press_3h); break; case WR_VAR(0, 10, 62): set_baro_sensor(var, sc::press_24h); break; case WR_VAR(0, 10, 63): set_baro_sensor(var, sc::press_tend); break; case WR_VAR(0, 10, 3): case WR_VAR(0, 10, 8): case WR_VAR(0, 10, 9): set_pressure(var); break; // Ship “instantaneous†data // Temperature and humidity data (complete) case WR_VAR(0, 12, 4): case WR_VAR(0, 12, 101): set_gen_sensor(var, sc::temp_2m); break; case WR_VAR(0, 12, 6): case WR_VAR(0, 12, 103): set_gen_sensor(var, sc::dewpoint_2m); break; case WR_VAR(0, 13, 3): set_gen_sensor(var, sc::humidity); break; case WR_VAR(0, 12, 2): case WR_VAR(0, 12, 102): set_gen_sensor(var, sc::wet_temp_2m); break; // Visibility data (complete) case WR_VAR(0, 20, 1): set_gen_sensor(var, sc::visibility); break; // Precipitation past 24h (complete) case WR_VAR(0, 13, 19): set_gen_sensor(var, sc::tot_prec1); break; case WR_VAR(0, 13, 20): set_gen_sensor(var, sc::tot_prec3); break; case WR_VAR(0, 13, 21): set_gen_sensor(var, sc::tot_prec6); break; case WR_VAR(0, 13, 22): set_gen_sensor(var, sc::tot_prec12); break; case WR_VAR(0, 13, 23): set_gen_sensor(var, sc::tot_prec24); break; // Cloud data case WR_VAR(0, 20, 10): set(var, sc::cloud_n); break; // Individual cloud layers or masses (complete) // Clouds with bases below station level (complete) // Direction of cloud drift (complete) case WR_VAR(0, 20, 11): case WR_VAR(0, 20, 13): case WR_VAR(0, 20, 17): case WR_VAR(0, 20, 54): set(var, var.code(), clouds.level, Trange::instant()); break; case WR_VAR(0, 20, 12): // CH CL CM set(var, WR_VAR(0, 20, 12), clouds.clcmch(), Trange::instant()); break; // Present and past weather (complete) case WR_VAR(0, 20, 3): set(var, sc::pres_wtr); break; case WR_VAR(0, 20, 4): set_past_weather(var, sc::past_wtr1_6h); break; case WR_VAR(0, 20, 5): set_past_weather(var, sc::past_wtr2_6h); break; // Precipitation measurement (complete) case WR_VAR(0, 13, 11): set_gen_sensor(var, WR_VAR(0, 13, 11), Level(1), Trange(1, 0, abs(trange.time_period))); break; // Extreme temperature data case WR_VAR(0, 12, 111): set_gen_sensor(var, WR_VAR(0, 12, 101), Level(1), Trange(2, -abs(trange.time_period_offset), abs(trange.time_period))); break; case WR_VAR(0, 12, 112): set_gen_sensor(var, WR_VAR(0, 12, 101), Level(1), Trange(3, -abs(trange.time_period_offset), abs(trange.time_period))); break; // Wind data (complete) case WR_VAR(0, 2, 2): set(var, sc::wind_inst); break; /* Note B/C 1.10.5.3.2 Calm shall be reported by * setting wind direction to 0 and wind speed to 0. * Variable shall be reported by setting wind direction * to 0 and wind speed to a positive value, not a * missing value indicator. */ case WR_VAR(0, 11, 1): case WR_VAR(0, 11, 11): set_wind(var, sc::wind_dir); break; case WR_VAR(0, 11, 2): case WR_VAR(0, 11, 12): set_wind(var, sc::wind_speed); break; case WR_VAR(0, 11, 43): set_wind_max(var, sc::wind_gust_max_dir); break; case WR_VAR(0, 11, 41): set_wind_max(var, sc::wind_gust_max_speed); break; case WR_VAR(0, 12, 5): set(var, sc::wet_temp_2m); break; case WR_VAR(0, 10,197): set(var, sc::height_anem); break; case WR_VAR(0, 12, 30): set(var, WR_VAR(0, 12, 30), Level(106, level.ground_depth == LevelContext::missing ? MISSING_INT : level.ground_depth * 1000), Trange::instant()); break; default: WMOImporter::import_var(var); break; } } } } } } dballe-8.6/dballe/msg/wr_importers/temp.cc0000644000175000017500000004567713554564112015605 00000000000000#include "base.h" #include #include #include #include #include #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include #include // Define to debug the sounding group matching algorithm // #define DEBUG_GROUPS #ifdef DEBUG_GROUPS #define debug_groups(...) fprintf(stderr, "grpmatch:" __VA_ARGS__) #else #define debug_groups(...) do {} while(0) #endif using namespace wreport; using namespace std; #define MISSING_PRESS -1.0 namespace dballe { namespace impl { namespace msg { namespace wr { class TempImporter : public WMOImporter { protected: double press; const Var* press_var; double surface_press; const Var* surface_press_var; void import_var(const Var& var); /** * If we identify sounding groups, this function can perform more accurate * sounding group import */ void import_group(unsigned start, unsigned length); public: TempImporter(const dballe::ImporterOptions& opts) : WMOImporter(opts) {} virtual ~TempImporter() {} virtual void init() { WMOImporter::init(); press = MISSING_PRESS; press_var = NULL; surface_press = MISSING_PRESS; surface_press_var = NULL; } /// Return true if \a code can be found at the start of a sounding group bool is_possible_group_var(Varcode code) { return WR_VAR_F(code) == 0 && WR_VAR_X(code) != 31 && WR_VAR_X(code) != 1; } /// If the next variables in the current subset look like \a group_count /// sounding groups, scan them and return true; otherwise return false. bool try_soundings(unsigned group_count) { // Check if the first var in the first candidate group is a likely start // of a sounding group if (pos + 1 == subset->size()) return false; const Var& start_var = (*subset)[pos + 1]; if (not is_possible_group_var(start_var.code())) return false; // start_var marks the start of a sounding group debug_groups("Candidate start var: %d (%01d%02d%03d) with group count %u\n", pos + 1, WR_VAR_F(start_var.code()), WR_VAR_X(start_var.code()), WR_VAR_Y(start_var.code()), group_count); // Seek forward until the same variable is found again, to compute the // group length unsigned group_length = 0; unsigned start = pos + 1; unsigned cur = start + 1; for ( ; cur < subset->size(); ++cur) { const Var& next = (*subset)[cur]; if (next.code() == start_var.code() || !is_possible_group_var(next.code())) { group_length = cur - start; if (start + group_count * group_length > subset->size()) return false; break; } } if (cur == subset->size()) group_length = cur-start; // Validate group_length checking that all groups start with start_var for (unsigned i = 0; i < group_count; ++i) { const Var& next = (*subset)[pos + 1 + i * group_length]; if (next.code() != start_var.code() && !is_possible_group_var(next.code())) { debug_groups(" Group count %u/%u fails check at leading var %01d%02d%03d\n", i, group_count, WR_VAR_F(next.code()), WR_VAR_X(next.code()), WR_VAR_Y(next.code())); return false; } } debug_groups("Validated first group: %d+%d\n", start, group_length); // Now we know how many groups there are and how long they are: iterate // them importing one at a time for (unsigned i = 0; i < group_count; ++i) import_group(pos + 1 + i * group_length, group_length); pos += 1 + group_count * group_length; return true; } virtual void run() { for (pos = 0; pos < subset->size(); ) { const Var& var = (*subset)[pos]; if (WR_VAR_F(var.code()) != 0 || !var.isset()) { // Ignore non-B variables and variables that are unset ++pos; } else if ((var.code() == WR_VAR(0, 31, 1) || var.code() == WR_VAR(0, 31, 2))) { // delayed descriptor replication count // Try to see if this is a sounding block, and import it a group // at a time if (!try_soundings(var.enqi())) // If it does not look like a sounding, ignore delayed // repetition count and proceed normally ++pos; } else { // Import all non-sounding vars import_var(var); ++pos; } } /* Extract surface data from the surface level */ if (surface_press != -1) { // Pressure is taken from a saved variable referencing to the original // pressure data in the message, to preserve data attributes if (surface_press_var && surface_press_var->isset() && msg->get_press_var()) msg->set_press_var(*surface_press_var); const Context* sfc = msg->find_context(Level(100, surface_press), Trange::instant()); if (sfc != NULL) { const Var* var = sfc->values.maybe_var(WR_VAR(0, 12, 1)); if (var && msg->get_temp_2m_var()) msg->set_temp_2m_var(*var); var = sfc->values.maybe_var(WR_VAR(0, 12, 3)); if (var && msg->get_dewpoint_2m_var()) msg->set_dewpoint_2m_var(*var); var = sfc->values.maybe_var(WR_VAR(0, 11, 1)); if (var && !msg->get_wind_dir_var()) msg->set_wind_dir_var(*var); var = sfc->values.maybe_var(WR_VAR(0, 11, 2)); if (var && !msg->get_wind_speed_var()) msg->set_wind_speed_var(*var); } } } MessageType scanType(const Bulletin& bulletin) const { switch (bulletin.data_category) { case 2: switch (bulletin.data_subcategory) { case 1: // 001 for PILOT data, case 2: // 002 for PILOT SHIP data, case 3: // 003 for PILOT MOBIL data. return MessageType::PILOT; case 4: return MessageType::TEMP; case 5: return MessageType::TEMP_SHIP; case 255: switch (bulletin.data_subcategory_local) { case 0: { /* Guess looking at the variables */ if (bulletin.subsets.empty()) throw error_consistency("trying to import a TEMP message with no data subset"); const Subset& subset = bulletin.subsets[0]; if (subset.size() > 1 && subset[0].code() == WR_VAR(0, 1, 11)) return MessageType::TEMP_SHIP; else return MessageType::TEMP; } case 101: return MessageType::TEMP; case 102: return MessageType::TEMP_SHIP; case 91: case 92: return MessageType::PILOT; } } break; case 6: return MessageType::TEMP; } return MessageType::TEMP; } }; std::unique_ptr Importer::createTemp(const dballe::ImporterOptions& opts) { return unique_ptr(new TempImporter(opts)); } void TempImporter::import_var(const Var& var) { switch (var.code()) { /* Identification of launch site and instrumentation */ case WR_VAR(0, 2, 3): msg->set_meas_equip_type_var(var); break; case WR_VAR(0, 2, 11): msg->set_sonde_type_var(var); break; case WR_VAR(0, 2, 12): msg->set_sonde_method_var(var); break; case WR_VAR(0, 2, 13): msg->set_sonde_correction_var(var); break; case WR_VAR(0, 2, 14): msg->set_sonde_tracking_var(var); break; /* Date/time of launch */ case WR_VAR(0, 8, 21): if (var.enqi() != 18) notes::log() << "TEMP time significance is " << var.enqi() << " instead of 18" << endl; break; /* Horizontal and vertical coordinates of launch site */ case WR_VAR(0, 7, 1): case WR_VAR(0, 7, 30): msg->set_height_station_var(var); break; case WR_VAR(0, 7, 31): msg->set_height_baro_var(var); break; case WR_VAR(0, 7, 7): msg->set_height_release_var(var); break; case WR_VAR(0, 33, 24): msg->set_station_height_quality_var(var); break; /* Cloud information reported with vertical soundings */ case WR_VAR(0, 8, 2): msg->set(Level::cloud(258, 0), Trange::instant(), WR_VAR(0, 8, 2), var); break; case WR_VAR(0, 20, 10): msg->set_cloud_n_var(var); break; case WR_VAR(0, 20, 11): msg->set_cloud_nh_var(var); break; case WR_VAR(0, 20, 13): msg->set_cloud_hh_var(var); break; case WR_VAR(0, 20, 12): { // CH CL CM int l2 = 1; if (pos > 0 && (*subset)[pos - 1].code() == WR_VAR(0, 20, 12)) { ++l2; if (pos > 1 && (*subset)[pos - 2].code() == WR_VAR(0, 20, 12)) ++l2; } msg->set(Level::cloud(258, l2), Trange::instant(), WR_VAR(0, 20, 12), var); break; } case WR_VAR(0, 22, 43): msg->set_water_temp_var(var); break; /* Temperature, dew-point and wind data at pressure levels */ // Long time period or displacement (since launch time) case WR_VAR(0, 4, 16): case WR_VAR(0, 4, 86): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 4, 86), var); break; // Extended vertical sounding significance case WR_VAR(0, 8, 42): { if (pos == subset->size() - 1) throw error_consistency("B08042 found at end of message"); if ((*subset)[pos + 1].code() == WR_VAR(0, 7, 4)) { // Pressure is reported later, we need to look ahead to compute the right level press_var = &((*subset)[pos + 1]); if (press_var->isset()) press = press_var->enqd(); else press = MISSING_INT; } msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 8, 42), var); break; } // Pressure case WR_VAR(0, 7, 4): press = var.enqd(); press_var = &var; msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 10, 4), var); break; // Vertical sounding significance case WR_VAR(0, 8, 1): { // This account for weird data that has '1' for VSS unsigned val = convert_BUFR08001_to_BUFR08042(var.enqi()); if (val != BUFR08042::ALL_MISSING) { unique_ptr nvar(newvar(WR_VAR(0, 8, 42), (int)val)); nvar->setattrs(var); msg->set(Level(100, press), Trange::instant(), move(nvar)); } } if (var.enqi() & BUFR08001::SURFACE) { surface_press = press; surface_press_var = press_var; } break; // Geopotential case WR_VAR(0, 10, 3): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 10, 8), var); break; case WR_VAR(0, 10, 8): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 10, 8), var); break; case WR_VAR(0, 10, 9): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 10, 8), var); break; // Latitude displacement case WR_VAR(0, 5, 15): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 5, 15), var); break; // Longitude displacement case WR_VAR(0, 6, 15): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 6, 15), var); break; // Dry bulb temperature case WR_VAR(0, 12, 1): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 12, 101), var); break; case WR_VAR(0, 12, 101): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 12, 101), var); break; // Wet bulb temperature case WR_VAR(0, 12, 2): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 12, 2), var); break; // Dew point temperature case WR_VAR(0, 12, 3): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 12, 103), var); break; case WR_VAR(0, 12, 103): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 12, 103), var); break; // Wind direction case WR_VAR(0, 11, 1): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 11, 1), var); break; // Wind speed case WR_VAR(0, 11, 2): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 11, 2), var); break; /* Wind shear data at a pressure level */ case WR_VAR(0, 11, 61): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 11, 61), var); break; case WR_VAR(0, 11, 62): msg->set(Level(100, press), Trange::instant(), WR_VAR(0, 11, 62), var); break; default: WMOImporter::import_var(var); break; } } void TempImporter::import_group(unsigned start, unsigned length) { // Compute vertical level information Level lev; for (unsigned i = 0; i < length && lev.ltype1 == MISSING_INT; ++i) { const Var& var = (*subset)[start + i]; switch (var.code()) { // Height level case WR_VAR(0, 7, 2): case WR_VAR(0, 7, 9): // Geopotential height, for pilots (FIXME: above ground or above msl?) if (var.isset()) lev = Level(102, var.enqd()); break; // Height level converted in mm case WR_VAR(0, 7, 7): if (var.isset()) lev = Level(102, var.enqd() * 1000); break; // Pressure level case WR_VAR(0, 7, 4): if (var.isset()) lev = Level(100, var.enqd()); break; case WR_VAR(0, 10, 3): // Convert geopotential to height if (var.isset()) lev = Level(102, lround(var.enqd() / 9.80665)); break; } } if (lev.ltype1 == MISSING_INT) return; //throw error_consistency("neither B07002 nor B07004 nor B10003 found in sounding group"); // Import all values for (unsigned i = 0; i < length; ++i) { const Var& var = (*subset)[start + i]; if (!var.isset()) { switch (var.code()) { case WR_VAR(0, 8, 1): case WR_VAR(0, 8, 42): // Preserve missing VSS with only the one missing bit set, // to act as a sounding context marker msg->set(lev, Trange::instant(), newvar(WR_VAR(0, 8, 42), (int)BUFR08042::MISSING)); break; } continue; } switch (var.code()) { case WR_VAR(0, 4, 16): case WR_VAR(0, 4, 86): msg->set(lev, Trange::instant(), WR_VAR(0, 4, 86), var); break; case WR_VAR(0, 5, 1): case WR_VAR(0, 5, 2): msg->set(lev, Trange::instant(), WR_VAR(0, 5, 1), var); break; case WR_VAR(0, 5, 15): msg->set(lev, Trange::instant(), WR_VAR(0, 5, 15), var); break; case WR_VAR(0, 6, 1): case WR_VAR(0, 6, 2): msg->set(lev, Trange::instant(), WR_VAR(0, 6, 1), var); break; case WR_VAR(0, 6, 15): msg->set(lev, Trange::instant(), WR_VAR(0, 6, 15), var); break; case WR_VAR(0, 8, 1): { // This accounts for weird data that has '1' for VSS unsigned val = convert_BUFR08001_to_BUFR08042(var.enqi()); if (val == BUFR08042::ALL_MISSING) msg->set(lev, Trange::instant(), newvar(WR_VAR(0, 8, 42), (int)BUFR08042::MISSING)); else { unique_ptr nvar(newvar(WR_VAR(0, 8, 42), (int)val)); nvar->setattrs(var); msg->set(lev, Trange::instant(), std::move(nvar)); } } break; case WR_VAR(0, 8, 42): msg->set(lev, Trange::instant(), WR_VAR(0, 8, 42), var); break; case WR_VAR(0, 10, 3): msg->set(lev, Trange::instant(), WR_VAR(0, 10, 8), var); break; case WR_VAR(0, 10, 9): msg->set(lev, Trange::instant(), WR_VAR(0, 10, 8), var); break; case WR_VAR(0, 12, 1): case WR_VAR(0, 12, 101): msg->set(lev, Trange::instant(), WR_VAR(0, 12, 101), var); break; case WR_VAR(0, 12, 3): case WR_VAR(0, 12, 103): msg->set(lev, Trange::instant(), WR_VAR(0, 12, 103), var); break; case WR_VAR(0, 7, 4): case WR_VAR(0, 10, 4): msg->set(lev, Trange::instant(), WR_VAR(0, 10, 4), var); break; case WR_VAR(0, 11, 1): case WR_VAR(0, 11, 2): msg->set(lev, Trange::instant(), var.code(), var); break; // Variables from Radar doppler wind profiles case WR_VAR(0, 11, 6): msg->set(lev, Trange::instant(), var.code(), var); break; case WR_VAR(0, 11, 50): msg->set(lev, Trange::instant(), var.code(), var); break; case WR_VAR(0, 33, 2): // Doppler wind profiles transmit quality information inline, // following the variable they refer to. if (i > 0) { // So we look at the varcode of the previous value Varcode prev_code = (*subset)[start + i - 1].code(); switch (prev_code) { // And if it is one of those for which quality info is // set in this way... case WR_VAR(0, 11, 2): case WR_VAR(0, 11, 6): // ...we lookup the variable we previously set in // msg and add the attribute to it if (wreport::Var* prev_var = msg->edit(prev_code, lev, Trange::instant())) prev_var->seta(var); break; } } break; //default: // msg->set(lev, Trange::instant(), var.code(), var); // break; } } } } } } } dballe-8.6/dballe/msg/wr_importers/ship.cc0000644000175000017500000000667113554573614015601 00000000000000#include "base.h" #include "dballe/msg/msg.h" #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { namespace { class ShipImporter : public SynopBaseImporter { protected: virtual void import_var(const Var& var); public: ShipImporter(const dballe::ImporterOptions& opts) : SynopBaseImporter(opts) {} virtual ~ShipImporter() {} MessageType scanType(const Bulletin& bulletin) const { switch (bulletin.data_category) { case 1: switch (bulletin.data_subcategory_local) { case 21: return MessageType::BUOY; case 9: case 11: case 12: case 13: case 14: case 19: return MessageType::SHIP; case 0: { // Guess looking at the variables if (bulletin.subsets.empty()) throw error_consistency("trying to import a SYNOP message with no data subset"); const Subset& subset = bulletin.subsets[0]; if (subset.size() > 1 && subset[0].code() == WR_VAR(0, 1, 5)) return MessageType::BUOY; else return MessageType::SHIP; } default: return MessageType::SHIP; } break; default: return MessageType::GENERIC; break; } } }; void ShipImporter::import_var(const Var& var) { switch (var.code()) { // Icing and ice case WR_VAR(0, 20, 31): case WR_VAR(0, 20, 32): case WR_VAR(0, 20, 33): case WR_VAR(0, 20, 34): case WR_VAR(0, 20, 35): case WR_VAR(0, 20, 36): case WR_VAR(0, 20, 37): case WR_VAR(0, 20, 38): msg->set(Level(1), Trange::instant(), var.code(), var); break; // Ship marine data case WR_VAR(0, 2, 38): msg->station_data.set(var.code(), var); break; case WR_VAR(0, 2, 39): msg->station_data.set(var.code(), var); break; case WR_VAR(0, 22, 42): case WR_VAR(0, 22, 43): if (level.sea_depth == LevelContext::missing) set(var, WR_VAR(0, 22, 43), Level(1), Trange::instant()); else set(var, WR_VAR(0, 22, 43), Level(161, level.sea_depth * 1000), Trange::instant()); break; // Waves case WR_VAR(0, 22, 1): case WR_VAR(0, 22, 11): case WR_VAR(0, 22, 21): case WR_VAR(0, 22, 2): case WR_VAR(0, 22, 12): case WR_VAR(0, 22, 22): msg->set(Level(1), Trange::instant(), var.code(), var); break; break; // D03023 swell waves (2 grups) case WR_VAR(0, 22, 3): // Direction of swell waves case WR_VAR(0, 22, 13): // Period of swell waves case WR_VAR(0, 22, 23): // Height of swell waves set(var, var.code(), Level(264, MISSING_INT, 261, level.swell_wave_group), Trange::instant()); break; default: SynopBaseImporter::import_var(var); break; } } } // anonynmous namespace std::unique_ptr Importer::createShip(const dballe::ImporterOptions& opts) { return unique_ptr(new ShipImporter(opts)); } } } } } dballe-8.6/dballe/msg/wr_importers/synop.cc0000644000175000017500000000616713554564112015777 00000000000000#include "base.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/msg.h" #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { namespace wr { namespace { class SynopImporter : public SynopBaseImporter { protected: virtual void import_var(const Var& var); public: SynopImporter(const dballe::ImporterOptions& opts) : SynopBaseImporter(opts) {} virtual ~SynopImporter() {} MessageType scanType(const Bulletin& bulletin) const { switch (bulletin.data_category) { case 0: return MessageType::SYNOP; default: return MessageType::GENERIC; break; } } }; void SynopImporter::import_var(const Var& var) { switch (var.code()) { // Direction and elevation of cloud (complete) case WR_VAR(0, 5, 21): set(var, WR_VAR(0, 5, 21), Level::cloud(262, 0), Trange::instant()); break; case WR_VAR(0, 7, 21): set(var, WR_VAR(0, 7, 21), Level::cloud(262, 0), Trange::instant()); break; // Cloud type is handled by the generic cloud type handler // State of ground, snow depth, ground minimum temperature (complete) case WR_VAR(0, 20, 62): set(var, sc::state_ground); break; case WR_VAR(0, 13, 13): set(var, sc::tot_snow); break; case WR_VAR(0, 12, 113): set(var, WR_VAR(0, 12, 121), Level(1), Trange(3, 0, 43200)); break; // Basic synoptic "period" data // Sunshine data (complete) case WR_VAR(0, 14, 31): set(var, WR_VAR(0, 14, 31), Level(1), Trange(1, 0, abs(trange.time_period))); break; // Evaporation data case WR_VAR(0, 2, 4): set(var, WR_VAR(0, 2, 4), Level(1), Trange::instant()); break; case WR_VAR(0, 13, 33): if (trange.time_period == MISSING_INT) set(var, WR_VAR(0, 13, 33), Level(1), Trange(1)); else set(var, WR_VAR(0, 13, 33), Level(1), Trange(1, 0, abs(trange.time_period))); break; // Radiation data case WR_VAR(0, 14, 2): case WR_VAR(0, 14, 4): case WR_VAR(0, 14, 16): case WR_VAR(0, 14, 28): case WR_VAR(0, 14, 29): case WR_VAR(0, 14, 30): if (trange.time_period == MISSING_INT) set(var, var.code(), Level(1), Trange(1)); else set(var, var.code(), Level(1), Trange(1, 0, abs(trange.time_period))); break; // Temperature change case WR_VAR(0, 12, 49): set(var, WR_VAR(0, 12, 49), Level(1), Trange(4, -abs(trange.time_period_offset), abs(trange.time_period))); break; case WR_VAR(0, 22, 42): set(var, sc::water_temp); break; case WR_VAR(0, 12, 5): set(var, sc::wet_temp_2m); break; case WR_VAR(0, 10,197): set(var, sc::height_anem); break; default: SynopBaseImporter::import_var(var); break; } } } // anonynmous namespace std::unique_ptr Importer::createSynop(const dballe::ImporterOptions& opts) { return unique_ptr(new SynopImporter(opts)); } } } } } dballe-8.6/dballe/msg/wr_importers/base.h0000644000175000017500000001556113554564112015401 00000000000000#ifndef DBALLE_MSG_WRIMPORTER_BASE_H #define DBALLE_MSG_WRIMPORTER_BASE_H #include #include #include #include namespace wreport { struct Subset; struct Bulletin; struct Var; } namespace dballe { namespace impl { namespace msg { namespace wr { class Importer { protected: const dballe::ImporterOptions& opts; const wreport::Subset* subset; impl::Message* msg; virtual void init(); virtual void run() = 0; void set(const wreport::Var& var, const Shortcut& shortcut); void set(const wreport::Var& var, wreport::Varcode code, const Level& level, const Trange& trange); public: Importer(const dballe::ImporterOptions& opts) : opts(opts) {} virtual ~Importer() {} virtual MessageType scanType(const wreport::Bulletin& bulletin) const = 0; void import(const wreport::Subset& subset, impl::Message& msg); static std::unique_ptr createSynop(const dballe::ImporterOptions&); static std::unique_ptr createShip(const dballe::ImporterOptions&); static std::unique_ptr createMetar(const dballe::ImporterOptions&); static std::unique_ptr createTemp(const dballe::ImporterOptions&); static std::unique_ptr createPilot(const dballe::ImporterOptions&); static std::unique_ptr createFlight(const dballe::ImporterOptions&); static std::unique_ptr createSat(const dballe::ImporterOptions&); static std::unique_ptr createPollution(const dballe::ImporterOptions&); static std::unique_ptr createGeneric(const dballe::ImporterOptions&); }; class WMOImporter : public Importer { protected: unsigned pos; void import_var(const wreport::Var& var); void init() override { pos = 0; Importer::init(); } public: WMOImporter(const dballe::ImporterOptions& opts) : Importer(opts) {} virtual ~WMOImporter() {} }; /// Keep track of level context changes struct LevelContext { static constexpr double missing = std::numeric_limits::max(); double height_baro; double press_std; double height_sensor; double sea_depth; double ground_depth; bool height_sensor_seen; bool swell_wave_group; void init(); void peek_var(const wreport::Var& var); }; /// Keep track of time range context changes struct TimerangeContext { int time_period; int time_period_offset; bool time_period_seen; int time_sig; int hour; int last_B04024_pos; void init(); void peek_var(const wreport::Var& var, unsigned pos); }; /** * Keep track of the current cloud metadata */ struct CloudContext { Level level; const Level& clcmch(); void init(); void on_vss(const wreport::Subset& subset, unsigned pos); }; /** * Check if the current context state of BUFR information is something that we * currently cannot handle. * * For example, a BUFR can provide a B12101 as a measured temperature, or a * B12101 as a standard deviation of temperature in the last 10 minutes. The * former we can handle, the latter we cannot. * * This class keeps track of when we are in such unusual states. * * See https://github.com/ARPA-SIMC/dballe/issues/47 */ struct UnsupportedContext { const wreport::Var* B08023 = nullptr; // First order statistics (code table) bool is_unsupported() const; void init(); void peek_var(const wreport::Var& var, unsigned pos); }; /** * Struct used to build an interpreted value */ struct Interpreted { /// Interpreted value being built std::unique_ptr var; /// Interpreted level Level level; /// Interpreted time range Trange trange; /** * Distance from the standard level to the real one. * * This is used, in case multiple values get simplified to the same level, * to select the one closer to the standard level. */ unsigned level_deviation = 0; /** * Beging building using a copy of var, and level and timerange from \a * shortcut */ Interpreted(const Shortcut& shortcut, const wreport::Var& var); Interpreted(const Shortcut& shortcut, const wreport::Var& var, const Level& level, const Trange& trange); Interpreted(wreport::Varcode code, const wreport::Var& var, const Level& level, const Trange& trange); virtual ~Interpreted(); virtual void set_sensor_height(const LevelContext& ctx) = 0; virtual void set_barometer_height(const LevelContext& ctx) = 0; virtual void set_duration(const TimerangeContext& ctx) = 0; virtual void set_wind_mean(const TimerangeContext& ctx) = 0; }; struct InterpretedPrecise : public Interpreted { using Interpreted::Interpreted; void set_sensor_height(const LevelContext& ctx) override; void set_barometer_height(const LevelContext& ctx) override; void set_duration(const TimerangeContext& ctx) override; void set_wind_mean(const TimerangeContext& ctx) override; }; struct InterpretedSimplified : public Interpreted { using Interpreted::Interpreted; void set_sensor_height(const LevelContext& ctx) override; void set_barometer_height(const LevelContext& ctx) override; void set_duration(const TimerangeContext& ctx) override; void set_wind_mean(const TimerangeContext& ctx) override; }; template std::unique_ptr create_interpreted(bool simplified, Args&& ...args) { if (simplified) return std::unique_ptr(new InterpretedSimplified(std::forward(args)...)); else return std::unique_ptr(new InterpretedPrecise(std::forward(args)...)); } /** * Base class for synop, ship and other importer with synop-like data */ class SynopBaseImporter : public WMOImporter { protected: CloudContext clouds; LevelContext level; TimerangeContext trange; UnsupportedContext unsupported; std::vector queued; virtual void peek_var(const wreport::Var& var); virtual void import_var(const wreport::Var& var); void set_gen_sensor(const wreport::Var& var, wreport::Varcode code, const Level& defaultLevel, const Trange& trange); void set_gen_sensor(const wreport::Var& var, const Shortcut& shortcut); void set_baro_sensor(const wreport::Var& var, const Shortcut& shortcut); void set_past_weather(const wreport::Var& var, const Shortcut& shortcut); void set_wind(const wreport::Var& var, const Shortcut& shortcut); void set_wind_max(const wreport::Var& var, const Shortcut& shortcut); void set_pressure(const wreport::Var& var); void set(const wreport::Var& var, const Shortcut& shortcut); void set(const wreport::Var& var, wreport::Varcode code, const Level& level, const Trange& trange); void set(std::unique_ptr val); public: SynopBaseImporter(const dballe::ImporterOptions& opts); ~SynopBaseImporter(); void init() override; void run() override; }; } } } } #endif dballe-8.6/dballe/msg/bulletin.cc0000644000175000017500000001070313554564112013700 00000000000000#include "bulletin.h" #include "dballe/core/csv.h" #include #include #include #include using namespace std; using namespace wreport; namespace dballe { namespace msg { namespace { /// Write CSV output to the given output stream struct Writer : public CSVWriter { FILE* out; Writer(FILE* out) : out(out) {} void flush_row() override { fputs(row.c_str(), out); putc('\n', out); row.clear(); } void print_var(const Var& var, const Var* parent=0) { string code; if (parent) { code += varcode_format(parent->code()); code += "."; } code += varcode_format(var.code()); if (var.isset()) { switch (var.info()->type) { case Vartype::Integer: add_keyval(code.c_str(), var.enqi()); break; case Vartype::Decimal: add_value(code.c_str()); add_value_raw(var.format("")); flush_row(); break; default: add_keyval(code.c_str(), var.format("")); break; } } else { add_value(code.c_str()); add_value_empty(); flush_row(); } } void print_subsets(const Bulletin& braw) { for (size_t i = 0; i < braw.subsets.size(); ++i) { const Subset& s = braw.subsets[i]; add_keyval("subset", i + 1); for (size_t i = 0; i < s.size(); ++i) { print_var(s[i]); for (const Var* a = s[i].next_attr(); a != NULL; a = a->next_attr()) print_var(*a, &(s[i])); } } } void add_keyval(const char* key, unsigned val) { add_value(key); add_value(val); flush_row(); } void add_keyval(const char* key, const std::string& val) { add_value(key); add_value(val); flush_row(); } void write_bulletin(const wreport::Bulletin& bul) { add_keyval("master_table_number", bul.master_table_number); add_keyval("data_category", bul.data_category); add_keyval("data_subcategory", bul.data_subcategory); add_keyval("data_subcategory_local", bul.data_subcategory_local); add_keyval("originating_centre", bul.originating_centre); add_keyval("originating_subcentre", bul.originating_subcentre); add_keyval("update_sequence_number", bul.update_sequence_number); char buf[30]; snprintf(buf, 29, "%04hu-%02hhu-%02hhu %02hhu:%02hhu:%02hhu", bul.rep_year, bul.rep_month, bul.rep_day, bul.rep_hour, bul.rep_minute, bul.rep_second); add_keyval("representative_time", buf); if (const BufrBulletin* b = dynamic_cast(&bul)) { add_keyval("encoding", "bufr"); add_keyval("edition_number", b->edition_number); add_keyval("master_table_version_number", b->master_table_version_number); add_keyval("master_table_version_number_local", b->master_table_version_number_local); add_keyval("compression", b->compression ? "true" : "false"); add_keyval("optional_section", str::encode_cstring(b->optional_section)); } else if (const CrexBulletin* b = dynamic_cast(&bul)) { add_keyval("encoding", "crex"); add_keyval("edition_number", b->edition_number); add_keyval("master_table_version_number", b->master_table_version_number); add_keyval("master_table_version_number_bufr", b->master_table_version_number_bufr); add_keyval("master_table_version_number_local", b->master_table_version_number_local); add_keyval("has_check_digit", b->has_check_digit ? "true" : "false"); } else throw error_consistency("encoding not supported for CSV dump"); print_subsets(bul); } }; } BulletinCSVWriter::BulletinCSVWriter(FILE* out) : out(out) { } BulletinCSVWriter::~BulletinCSVWriter() { } void BulletinCSVWriter::output_bulletin(const wreport::Bulletin& bulletin) { Writer writer(out); // Print column titles at the first BUFR if (first) { writer.add_value("Field"); writer.add_value("Value"); writer.flush_row(); first = false; } writer.write_bulletin(bulletin); } } } dballe-8.6/dballe/msg/cursor.cc0000644000175000017500000000030113554564112013370 00000000000000#include "cursor.h" namespace dballe { namespace impl { namespace msg { CursorStation::~CursorStation() { } CursorStationData::~CursorStationData() { } CursorData::~CursorData() { } } } } dballe-8.6/dballe/msg/cursor-access.cc0000644000175000017500000004305213554564124014644 00000000000000#include "cursor.h" #include using namespace wreport; namespace dballe { namespace impl { namespace msg { void CursorStation::enq(impl::Enq& enq) const { if (enq.search_b_values(station_values)) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 3: if (key[0] == 'l') { switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(station.coords.lat); } else { enq.search_alias_values(station_values); } break; case 'o': if (key[2] == 'n') { enq.set_lon(station.coords.lon); } else { enq.search_alias_values(station_values); } break; default: enq.search_alias_values(station_values); } } else { enq.search_alias_values(station_values); } break; case 5: if (memcmp(key + 0, "ident", 5) == 0) { enq.set_ident(station.ident); } else { enq.search_alias_values(station_values); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(station.report); } else { enq.search_alias_values(station_values); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(station.id); } else { enq.search_alias_values(station_values); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!station.ident.is_missing()); } else { enq.search_alias_values(station_values); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(station.coords); } else { enq.search_alias_values(station_values); } break; default: enq.search_alias_values(station_values); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(station); } else { enq.search_alias_values(station_values); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { return; } else { enq.search_alias_values(station_values); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(station.report); } else { enq.search_alias_values(station_values); } break; default: enq.search_alias_values(station_values); } break; default: enq.search_alias_values(station_values); } } void CursorStationData::enq(impl::Enq& enq) const { if (enq.search_b_value(*cur)) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(station.coords.lat); } else { enq.search_alias_value(*cur); } break; case 'o': if (key[2] == 'n') { enq.set_lon(station.coords.lon); } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->code()); } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(station.ident); } else { enq.search_alias_value(*cur); } break; case 'a': if (memcmp(key + 1, "ttrs", 4) == 0) { enq.set_attrs(cur->get()); } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(station.report); } else { enq.search_alias_value(*cur); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(station.id); } else { enq.search_alias_value(*cur); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!station.ident.is_missing()); } else { enq.search_alias_value(*cur); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(station.coords); } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(station); } else { enq.search_alias_value(*cur); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { return; } else { enq.search_alias_value(*cur); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(station.report); } else { enq.search_alias_value(*cur); } break; case 'v': if (memcmp(key + 1, "ariable", 7) == 0) { enq.set_var(cur->get()); } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } break; case 10: if (memcmp(key + 0, "context_id", 10) == 0) { return; } else { enq.search_alias_value(*cur); } break; default: enq.search_alias_value(*cur); } } void CursorData::enq(impl::Enq& enq) const { if (enq.search_b_value(*(cur->var))) return; const auto key = enq.key; const auto len = enq.len; switch (len) { case 2: switch (key[0]) { case 'l': switch (key[1]) { case '1': enq.set_dballe_int(cur->level.l1); break; case '2': enq.set_dballe_int(cur->level.l2); break; default: enq.search_alias_value(*(cur->var)); } break; case 'p': switch (key[1]) { case '1': enq.set_dballe_int(cur->trange.p1); break; case '2': enq.set_dballe_int(cur->trange.p2); break; default: enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 3: switch (key[0]) { case 'l': switch (key[1]) { case 'a': if (key[2] == 't') { enq.set_lat(station.coords.lat); } else { enq.search_alias_value(*(cur->var)); } break; case 'o': if (key[2] == 'n') { enq.set_lon(station.coords.lon); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 'd': if (memcmp(key + 1, "ay", 2) == 0) { enq.set_int(datetime.day); } else { enq.search_alias_value(*(cur->var)); } break; case 'm': if (memcmp(key + 1, "in", 2) == 0) { enq.set_int(datetime.minute); } else { enq.search_alias_value(*(cur->var)); } break; case 's': if (memcmp(key + 1, "ec", 2) == 0) { enq.set_int(datetime.second); } else { enq.search_alias_value(*(cur->var)); } break; case 'v': if (memcmp(key + 1, "ar", 2) == 0) { enq.set_varcode(cur->var->code()); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 4: switch (key[0]) { case 'y': if (memcmp(key + 1, "ear", 3) == 0) { enq.set_int(datetime.year); } else { enq.search_alias_value(*(cur->var)); } break; case 'h': if (memcmp(key + 1, "our", 3) == 0) { enq.set_int(datetime.hour); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 5: switch (key[0]) { case 'i': if (memcmp(key + 1, "dent", 4) == 0) { enq.set_ident(station.ident); } else { enq.search_alias_value(*(cur->var)); } break; case 'm': if (memcmp(key + 1, "onth", 4) == 0) { enq.set_int(datetime.month); } else { enq.search_alias_value(*(cur->var)); } break; case 'l': if (memcmp(key + 1, "evel", 4) == 0) { enq.set_level(cur->level); } else { enq.search_alias_value(*(cur->var)); } break; case 'a': if (memcmp(key + 1, "ttrs", 4) == 0) { enq.set_attrs(cur->var->get()); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 6: switch (key[0]) { case 'r': if (memcmp(key + 1, "eport", 5) == 0) { enq.set_string(station.report); } else { enq.search_alias_value(*(cur->var)); } break; case 'a': if (memcmp(key + 1, "na_id", 5) == 0) { enq.set_dballe_int(station.id); } else { enq.search_alias_value(*(cur->var)); } break; case 'm': if (memcmp(key + 1, "obile", 5) == 0) { enq.set_bool(!station.ident.is_missing()); } else { enq.search_alias_value(*(cur->var)); } break; case 'c': if (memcmp(key + 1, "oords", 5) == 0) { enq.set_coords(station.coords); } else { enq.search_alias_value(*(cur->var)); } break; case 't': if (memcmp(key + 1, "range", 5) == 0) { enq.set_trange(cur->trange); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 7: if (memcmp(key + 0, "station", 7) == 0) { enq.set_station(station); } else { enq.search_alias_value(*(cur->var)); } break; case 8: switch (key[0]) { case 'p': if (memcmp(key + 1, "riority", 7) == 0) { return; } else { enq.search_alias_value(*(cur->var)); } break; case 'r': if (memcmp(key + 1, "ep_memo", 7) == 0) { enq.set_string(station.report); } else { enq.search_alias_value(*(cur->var)); } break; case 'd': if (memcmp(key + 1, "atetime", 7) == 0) { enq.set_datetime(datetime); } else { enq.search_alias_value(*(cur->var)); } break; case 'v': if (memcmp(key + 1, "ariable", 7) == 0) { enq.set_var(cur->var->get()); } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; case 10: switch (key[0]) { case 'l': if (memcmp(key + 1, "eveltype", 8) == 0) { switch (key[9]) { case '1': enq.set_dballe_int(cur->level.ltype1); break; case '2': enq.set_dballe_int(cur->level.ltype2); break; default: enq.search_alias_value(*(cur->var)); } } else { enq.search_alias_value(*(cur->var)); } break; case 'p': if (memcmp(key + 1, "indicator", 9) == 0) { enq.set_dballe_int(cur->trange.pind); } else { enq.search_alias_value(*(cur->var)); } break; case 'c': if (memcmp(key + 1, "ontext_id", 9) == 0) { return; } else { enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } break; default: enq.search_alias_value(*(cur->var)); } } } } } dballe-8.6/dballe/msg/wr_codec.cc0000644000175000017500000003564313554564112013661 00000000000000#include "wr_codec.h" #include "dballe/file.h" #include "msg.h" #include "context.h" #include "dballe/core/shortcuts.h" #include "wr_importers/base.h" #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { WRImporter::WRImporter(const dballe::ImporterOptions& opts) : Importer(opts) {} BufrImporter::BufrImporter(const dballe::ImporterOptions& opts) : WRImporter(opts) {} BufrImporter::~BufrImporter() {} bool BufrImporter::foreach_decoded(const BinaryMessage& msg, std::function)> dest) const { unique_ptr bulletin(BufrBulletin::decode(msg.data)); return foreach_decoded_bulletin(*bulletin, dest); } CrexImporter::CrexImporter(const dballe::ImporterOptions& opts) : WRImporter(opts) {} CrexImporter::~CrexImporter() {} bool CrexImporter::foreach_decoded(const BinaryMessage& msg, std::function)> dest) const { unique_ptr bulletin(CrexBulletin::decode(msg.data)); return foreach_decoded_bulletin(*bulletin, dest); } Messages WRImporter::from_bulletin(const wreport::Bulletin& msg) const { Messages res; foreach_decoded_bulletin(msg, [&](unique_ptr&& m) { res.emplace_back(move(m)); return true; }); return res; } bool WRImporter::foreach_decoded_bulletin(const wreport::Bulletin& msg, std::function)> dest) const { // Infer the right importer. See Common Code Table C-13 std::unique_ptr importer; switch (msg.data_category) { // Surface data - land case 0: switch (msg.data_subcategory) { // Routine aeronautical observations (METAR) case 10: importer = wr::Importer::createMetar(opts); break; default: // Old ECMWF METAR type if (msg.data_subcategory_local == 140) importer = wr::Importer::createMetar(opts); else importer = wr::Importer::createSynop(opts); break; } break; // Surface data - sea case 1: importer = wr::Importer::createShip(opts); break; // Vertical soundings (other than satellite) case 2: importer = wr::Importer::createTemp(opts); break; // Vertical soundings (satellite) case 3: importer = wr::Importer::createSat(opts); break; // Single level upper-air data (other than satellite) case 4: importer = wr::Importer::createFlight(opts); break; // Radar data case 6: if (msg.data_subcategory == 1) // Doppler wind profiles importer = wr::Importer::createTemp(opts); else importer = wr::Importer::createGeneric(opts); break; // Physical/chemical constituents case 8: importer = wr::Importer::createPollution(opts); break; default: importer = wr::Importer::createGeneric(opts); break; } MessageType type = importer->scanType(msg); for (unsigned i = 0; i < msg.subsets.size(); ++i) { std::unique_ptr newmsg(new Message); newmsg->type = type; importer->import(msg.subsets[i], *newmsg); if (!dest(move(newmsg))) return false; } return true; } WRExporter::WRExporter(const dballe::ExporterOptions& opts) : Exporter(opts) {} BufrExporter::BufrExporter(const dballe::ExporterOptions& opts) : WRExporter(opts) {} BufrExporter::~BufrExporter() {} std::unique_ptr BufrExporter::make_bulletin() const { return std::unique_ptr(BufrBulletin::create().release()); } std::string BufrExporter::to_binary(const Messages& msgs) const { return to_bulletin(msgs)->encode(); } CrexExporter::CrexExporter(const dballe::ExporterOptions& opts) : WRExporter(opts) {} CrexExporter::~CrexExporter() {} std::unique_ptr CrexExporter::make_bulletin() const { return std::unique_ptr(CrexBulletin::create().release()); } std::string CrexExporter::to_binary(const Messages& msgs) const { return to_bulletin(msgs)->encode(); } namespace { const char* infer_from_message(const Message& msg) { switch (msg.type) { case MessageType::TEMP_SHIP: return "temp-ship"; default: break; } return format_message_type(msg.type); } } unique_ptr WRExporter::infer_template(const Messages& msgs) const { // Select initial template name string tpl = opts.template_name; if (tpl.empty()) tpl = infer_from_message(Message::downcast(*msgs[0])); // Get template factory const wr::TemplateFactory& fac = wr::TemplateRegistry::get(tpl); return fac.factory(opts, msgs); } unique_ptr WRExporter::to_bulletin(const Messages& msgs) const { std::unique_ptr encoder = infer_template(msgs); // fprintf(stderr, "Encoding with template %s\n", encoder->name()); auto res = make_bulletin(); encoder->to_bulletin(*res); return res; } namespace wr { extern void register_synop(TemplateRegistry&); extern void register_ship(TemplateRegistry&); extern void register_buoy(TemplateRegistry&); extern void register_metar(TemplateRegistry&); extern void register_temp(TemplateRegistry&); extern void register_flight(TemplateRegistry&); extern void register_generic(TemplateRegistry&); extern void register_pollution(TemplateRegistry&); static TemplateRegistry* registry = NULL; const TemplateRegistry& TemplateRegistry::get() { if (!registry) { registry = new TemplateRegistry; registry->register_factory(MISSING_INT, "wmo", "WMO style templates (autodetect)", [](const dballe::ExporterOptions& opts, const Messages& msgs) { auto msg = Message::downcast(msgs[0]); string tpl; switch (msg->type) { case MessageType::TEMP_SHIP: tpl = "temp-wmo"; break; default: tpl = format_message_type(msg->type); tpl += "-wmo"; break; } const wr::TemplateFactory& fac = wr::TemplateRegistry::get(tpl); return fac.factory(opts, msgs); }); // Populate it register_synop(*registry); register_ship(*registry); register_buoy(*registry); register_metar(*registry); register_temp(*registry); register_flight(*registry); register_generic(*registry); register_pollution(*registry); // registry->insert("synop", ...) // registry->insert("synop-high", ...) // registry->insert("wmo-synop", ...) // registry->insert("wmo-synop-high", ...) // registry->insert("ecmwf-synop", ...) // registry->insert("ecmwf-synop-high", ...) } return *registry; } const TemplateFactory& TemplateRegistry::get(const std::string& name) { const TemplateRegistry& tr = get(); TemplateRegistry::const_iterator i = tr.find(name); if (i == tr.end()) error_notfound::throwf("requested export template %s which does not exist", name.c_str()); return i->second; } void TemplateRegistry::register_factory( unsigned data_category, const std::string& name, const std::string& desc, TemplateFactory::factory_func fac) { insert(make_pair(name, TemplateFactory(data_category, name, desc, fac))); } void Template::to_bulletin(wreport::Bulletin& bulletin) { setupBulletin(bulletin); for (unsigned i = 0; i < msgs.size(); ++i) { Subset& s = bulletin.obtain_subset(i); to_subset(Message::downcast(*msgs[i]), s); } } void Template::setupBulletin(wreport::Bulletin& bulletin) { // Get reference time from first msg in the set // If not found, use current time. Datetime dt = msgs[0]->get_datetime(); bulletin.rep_year = dt.year; bulletin.rep_month = dt.month; bulletin.rep_day = dt.day; bulletin.rep_hour = dt.hour; bulletin.rep_minute = dt.minute; bulletin.rep_second = dt.second; bulletin.master_table_number = 0; bulletin.originating_centre = opts.centre != MISSING_INT ? opts.centre : 255; bulletin.originating_subcentre = opts.subcentre != MISSING_INT ? opts.subcentre : 255; bulletin.update_sequence_number = 0; if (BufrBulletin* b = dynamic_cast(&bulletin)) { // Take from opts b->edition_number = 4; b->master_table_version_number = 17; b->master_table_version_number_local = 0; b->compression = false; } if (CrexBulletin* b = dynamic_cast(&bulletin)) { // TODO: change using BUFR tables, when the encoder can encode the full // CREX ed.2 header b->edition_number = 2; b->master_table_version_number = 3; b->master_table_version_number_local = 0; b->master_table_version_number_bufr = 0; b->has_check_digit = false; } } void Template::to_subset(const Message& msg, wreport::Subset& subset) { this->msg = &msg; this->subset = ⊂ this->c_gnd_instant = msg.find_context(Level(1), Trange::instant()); } void Template::add(Varcode code, const msg::Context* ctx, const Shortcut& shortcut) const { if (!ctx) subset->store_variable_undef(code); else if (const Var* var = ctx->values.maybe_var(shortcut.code)) subset->store_variable(code, *var); else subset->store_variable_undef(code); } void Template::add(Varcode code, const msg::Context* ctx, Varcode srccode) const { if (!ctx) subset->store_variable_undef(code); else if (const Var* var = ctx->values.maybe_var(srccode)) subset->store_variable(code, *var); else subset->store_variable_undef(code); } void Template::add(Varcode code, const msg::Context* ctx) const { if (!ctx) subset->store_variable_undef(code); else add(code, ctx->values); } void Template::add(Varcode code, const Values& values) const { if (const Var* var = values.maybe_var(code)) subset->store_variable(*var); else subset->store_variable_undef(code); } void Template::add(Varcode code, const Values& values, const Shortcut& shortcut) const { if (const Var* var = values.maybe_var(shortcut.code)) subset->store_variable(code, *var); else subset->store_variable_undef(code); } void Template::add(Varcode code, const Shortcut& shortcut) const { add(code, msg->get(shortcut)); } void Template::add(Varcode code, Varcode srccode, const Level& level, const Trange& trange) const { add(code, msg->get(level, trange, srccode)); } void Template::add(wreport::Varcode code, const wreport::Var* var) const { if (var) subset->store_variable(code, *var); else subset->store_variable_undef(code); } const Var* Template::find_station_var(wreport::Varcode code) const { return msg->station_data.maybe_var(code); } void Template::do_station_name(wreport::Varcode dstcode) const { if (const wreport::Var* var = msg->station_data.maybe_var(sc::st_name.code)) { Varinfo info = subset->tables->btable->query(dstcode); Var name(info); if (var->isset()) name.setc_truncate(var->enqc()); subset->store_variable(move(name)); } else subset->store_variable_undef(dstcode); } void Template::do_ecmwf_past_wtr() const { int hour = msg->get_datetime().hour == 0xff ? 0 : msg->get_datetime().hour; if (hour % 6 == 0) { add(WR_VAR(0, 20, 4), sc::past_wtr1_6h); add(WR_VAR(0, 20, 5), sc::past_wtr2_6h); } else { add(WR_VAR(0, 20, 4), sc::past_wtr1_3h); add(WR_VAR(0, 20, 5), sc::past_wtr2_3h); } } void Template::do_station_height() const { add(WR_VAR(0, 7, 30), msg->station_data); add(WR_VAR(0, 7, 31), msg->station_data); } void Template::do_D01001() const { add(WR_VAR(0, 1, 1), msg->station_data, sc::block); add(WR_VAR(0, 1, 2), msg->station_data, sc::station); } void Template::do_D01004() const { do_D01001(); do_station_name(WR_VAR(0, 1, 15)); add(WR_VAR(0, 2, 1), msg->station_data, sc::st_type); } void Template::do_D01011() const { // Year if (const Var* var = find_station_var(WR_VAR(0, 4, 1))) subset->store_variable(WR_VAR(0, 4, 1), *var); else if (!msg->get_datetime().is_missing()) subset->store_variable_i(WR_VAR(0, 4, 1), msg->get_datetime().year); else subset->store_variable_undef(WR_VAR(0, 4, 1)); // Month if (const Var* var = find_station_var(WR_VAR(0, 4, 2))) subset->store_variable(WR_VAR(0, 4, 2), *var); else if (!msg->get_datetime().is_missing()) subset->store_variable_i(WR_VAR(0, 4, 2), msg->get_datetime().month); else subset->store_variable_undef(WR_VAR(0, 4, 2)); // Day if (const Var* var = find_station_var(WR_VAR(0, 4, 3))) subset->store_variable(WR_VAR(0, 4, 3), *var); else if (!msg->get_datetime().is_missing()) subset->store_variable_i(WR_VAR(0, 4, 3), msg->get_datetime().day); else subset->store_variable_undef(WR_VAR(0, 4, 3)); } int Template::do_D01012() const { int res = MISSING_INT; // Hour if (const Var* var = find_station_var(WR_VAR(0, 4, 4))) { subset->store_variable(WR_VAR(0, 4, 4), *var); res = var->enqi(); } else if (!msg->get_datetime().is_missing()) { subset->store_variable_i(WR_VAR(0, 4, 4), msg->get_datetime().hour); res = msg->get_datetime().hour; } else subset->store_variable_undef(WR_VAR(0, 4, 4)); // Minute if (const Var* var = find_station_var(WR_VAR(0, 4, 5))) subset->store_variable(WR_VAR(0, 4, 5), *var); else if (!msg->get_datetime().is_missing()) subset->store_variable_i(WR_VAR(0, 4, 5), msg->get_datetime().minute); else subset->store_variable_undef(WR_VAR(0, 4, 5)); return res; } void Template::do_D01013() const { do_D01012(); // Second if (const Var* var = find_station_var(WR_VAR(0, 4, 6))) subset->store_variable(WR_VAR(0, 4, 6), *var); else if (!msg->get_datetime().is_missing()) subset->store_variable_i(WR_VAR(0, 4, 6), msg->get_datetime().second); else subset->store_variable_i(WR_VAR(0, 4, 6), 0); } void Template::do_D01021() const { add(WR_VAR(0, 5, 1), msg->station_data, sc::latitude); add(WR_VAR(0, 6, 1), msg->station_data, sc::longitude); } void Template::do_D01022() const { do_D01021(); add(WR_VAR(0, 7, 1), msg->station_data, sc::height_station); } void Template::do_D01023() const { add(WR_VAR(0, 5, 2), msg->station_data, sc::latitude); add(WR_VAR(0, 6, 2), msg->station_data, sc::longitude); } } } } } dballe-8.6/dballe/msg/tests.h0000644000175000017500000002154613554573614013104 00000000000000#include #include #include #include #include #include namespace wreport { struct Vartable; } namespace dballe { namespace tests { impl::Messages read_msgs(const char* filename, Encoding type, const dballe::ImporterOptions& opts=dballe::ImporterOptions::defaults); impl::Messages read_msgs(const char* filename, Encoding type, const std::string& opts); impl::Messages read_msgs_csv(const char* filename); struct ActualMessage : public Actual { using Actual::Actual; void is_undef(const impl::Shortcut& shortcut) const; }; inline ActualMessage actual(const Message& message) { return ActualMessage(message); } std::unique_ptr export_msgs(Encoding enctype, const impl::Messages& in, const std::string& tag, const dballe::ExporterOptions& opts=dballe::ExporterOptions::defaults); #define test_export_msgs(...) wcallchecked(export_msgs(__VA_ARGS__)) void track_different_msgs(const Message& msg1, const Message& msg2, const std::string& prefix); void track_different_msgs(const impl::Messages& msgs1, const impl::Messages& msgs2, const std::string& prefix); extern const char* bufr_files[]; extern const char* crex_files[]; const wreport::Var& want_var(const Message& msg, const impl::Shortcut& shortcut); const wreport::Var& want_var(const Message& msg, wreport::Varcode code, const dballe::Level& lev, const dballe::Trange& tr); inline ActualVar actual_var(const Message& message, const impl::Shortcut& shortcut) { return ActualVar(want_var(message, shortcut)); } inline ActualVar actual_var(const Message& message, wreport::Varcode code, const dballe::Level& lev, const dballe::Trange& tr) { return ActualVar(want_var(message, code, lev, tr)); } void dump(const std::string& tag, const Message& msg, const std::string& desc="message"); void dump(const std::string& tag, const impl::Messages& msgs, const std::string& desc="message"); void dump(const std::string& tag, const wreport::Bulletin& bul, const std::string& desc="message"); void dump(const std::string& tag, const BinaryMessage& msg, const std::string& desc="message"); void dump(const std::string& tag, const std::string& str, const std::string& desc="message"); struct MessageTweaker { virtual ~MessageTweaker() {} virtual void tweak(impl::Messages&) {} virtual std::string desc() const = 0; }; struct MessageTweakers { std::vector tweaks; ~MessageTweakers(); // Takes ownership of memory management void add(MessageTweaker* tweak); void apply(impl::Messages& msgs); }; namespace tweaks { // Strip attributes from all variables in a impl::Messages struct StripAttrs : public MessageTweaker { std::vector codes; void tweak(impl::Messages& msgs); virtual std::string desc() const { return "StripAttrs"; } }; // Strip attributes from all variables in a impl::Messages struct StripQCAttrs : public StripAttrs { StripQCAttrs(); virtual std::string desc() const { return "StripQCAttrs"; } }; // Strip attributes with substituted values struct StripSubstituteAttrs : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "StripSubstituteAttrs"; } }; // Strip context attributes from all variables in a impl::Messages struct StripContextAttrs : public StripAttrs { StripContextAttrs(); virtual std::string desc() const { return "StripContextAttrs"; } }; // Strip a user-defined list of vars from all levels struct StripVars : public MessageTweaker { std::vector codes; StripVars() {} StripVars(std::initializer_list codes) : codes(codes) {} void tweak(impl::Messages& msgs); virtual std::string desc() const { return "StripVars"; } }; // Round variables to account for a passage through legacy vars struct RoundLegacyVars : public MessageTweaker { const wreport::Vartable* table; RoundLegacyVars(); void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RoundLegacyVars"; } }; // Remove synop vars present in WMO templates but not in ECMWF templates struct RemoveSynopWMOOnlyVars : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RemoveSynopWMOOnlyVars"; } }; // Remove temp vars present in WMO templates but not in ECMWF templates struct RemoveTempWMOOnlyVars : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RemoveTempWMOOnlyVars"; } }; // Remove temp vars present only in an odd temp template for which we have // messages in the test suite struct RemoveOddTempTemplateOnlyVars : public StripVars { RemoveOddTempTemplateOnlyVars(); virtual std::string desc() const { return "RemoveOddTempTemplateOnlyVars"; } }; // Remove ground level with missing length of statistical processing, that // cannot be encoded in ECMWF templates struct RemoveSynopWMOOddprec : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RemoveSynopWMOOddprec"; } }; // Truncate station name to its canonical length struct TruncStName : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "TruncStName"; } }; // Round geopotential with a B10003->B10008->B10009->B10008->B10003 round trip struct RoundGeopotential : public MessageTweaker { const wreport::Vartable* table; RoundGeopotential(); void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RoundGeopotential"; } }; // Add B10008 GEOPOTENTIAL to all height levels, with its value taken from the height struct HeightToGeopotential : public MessageTweaker { const wreport::Vartable* table; HeightToGeopotential(); void tweak(impl::Messages& msgs); virtual std::string desc() const { return "HeightToGeopotential"; } }; // Round vertical sounding significance with a B08042->B08001->B08042 round trip struct RoundVSS : public MessageTweaker { void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RoundVSS"; } }; // Remove a context given its level and time range struct RemoveContext : public MessageTweaker { Level lev; Trange tr; RemoveContext(const Level& lev, const Trange& tr); void tweak(impl::Messages& msgs); virtual std::string desc() const { return "RemoveContext"; } }; } struct TestMessage { std::string name; Encoding type; BinaryMessage raw; wreport::Bulletin* bulletin = 0; impl::Messages msgs; TestMessage(Encoding type, const std::string& name); ~TestMessage(); void read_from_file(const std::string& fname, const ImporterOptions& input_opts); void read_from_raw(const BinaryMessage& msg, const ImporterOptions& input_opts); void read_from_msgs(const impl::Messages& msgs, const ExporterOptions& export_opts); void dump() const; }; struct TestCodec { std::string fname; Encoding type; bool verbose = false; impl::ImporterOptions input_opts; impl::ExporterOptions output_opts; std::string expected_template; int expected_subsets = 1; int expected_min_vars = 1; int expected_data_category = MISSING_INT; int expected_data_subcategory = MISSING_INT; int expected_data_subcategory_local = MISSING_INT; MessageTweakers after_reimport_import; MessageTweakers after_reimport_reimport; MessageTweakers after_convert_import; MessageTweakers after_convert_reimport; void do_compare(const TestMessage& msg1, const TestMessage& msg2); TestCodec(const std::string& fname, Encoding type=Encoding::BUFR); void configure_ecmwf_to_wmo_tweaks(); // "import, export, import again, compare" test void run_reimport(); // "import, export as different template, import again, compare" test void run_convert(const std::string& tplname); }; #if 0 /* Random message generation functions */ class msg_generator : public generator { public: dba_err fill_message(dba_msg msg, bool mobile); }; /* Message reading functions */ class msg_vector : public dba_raw_consumer, public std::vector { public: virtual ~msg_vector() { for (iterator i = begin(); i != end(); i++) dba_msgs_delete(*i); } virtual dba_err consume(dba_rawmsg raw) { dba_msgs msgs; DBA_RUN_OR_RETURN(dba_marshal_decode(raw, &msgs)); push_back(msgs); return dba_error_ok(); } }; template void my_ensure_msg_equals(const char* file, int line, dba_msg msg, int id, const char* idname, const T& value) { dba_var var = my_want_var(file, line, msg, id, idname); inner_ensure_var_equals(var, value); } #define gen_ensure_msg_equals(msg, id, value) my_ensure_msg_equals(__FILE__, __LINE__, (msg), (id), #id, (value)) #define inner_ensure_msg_equals(msg, id, value) my_ensure_msg_equals(file, line, (msg), (id), #id, (value)) #endif } } // vim:set ts=4 sw=4: dballe-8.6/dballe/msg/cursor.h0000644000175000017500000001410613572414716013246 00000000000000#ifndef DBALLE_MSG_CURSOR_H #define DBALLE_MSG_CURSOR_H #include #include #include #include namespace dballe { namespace impl { namespace msg { struct CursorStation : public impl::CursorStation { dballe::DBStation station; const Values& station_values; bool at_start = true; CursorStation(const impl::Message& msg) : station_values(msg.find_station_context()) { station.report = msg.get_report(); station.coords = msg.get_coords(); station.ident = msg.get_ident(); } ~CursorStation(); bool has_value() const override { return !at_start; } int remaining() const override { if (at_start) return 1; return 0; } bool next() override { if (at_start) { at_start = false; return true; } else return false; } void discard() override { at_start = false; } void enq(Enq& enq) const override; DBStation get_station() const override { return station; } DBValues get_values() const override { return DBValues(station_values); } /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { CursorStation* res = dynamic_cast(c.get()); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of cursor"); c.release(); return std::unique_ptr(res); } }; struct CursorStationData : public impl::CursorStationData { dballe::DBStation station; const Values& station_values; bool at_start = true; Values::const_iterator cur; CursorStationData(const impl::Message& msg) : station_values(msg.find_station_context()) { station.report = msg.get_report(); station.coords = msg.get_coords(); station.ident = msg.get_ident(); } ~CursorStationData(); bool has_value() const override { return !at_start && cur != station_values.end(); } int remaining() const override { if (at_start) return station_values.size(); return station_values.end() - cur; } bool next() override { if (at_start) { at_start = false; cur = station_values.begin(); return true; } else if (cur == station_values.end()) return false; else { ++cur; return cur != station_values.end(); } } void discard() override { at_start = false; cur = station_values.end(); } void enq(Enq& enq) const override; DBStation get_station() const override { return station; } wreport::Varcode get_varcode() const override { return (*cur)->code(); } wreport::Var get_var() const override { return **cur; } /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { CursorStationData* res = dynamic_cast(c.get()); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of cursor"); c.release(); return std::unique_ptr(res); } }; struct CursorDataRow { Level level; Trange trange; Values::const_iterator var; CursorDataRow(Values::const_iterator var) : var(var) { } CursorDataRow(const Level& level, const Trange& trange, Values::const_iterator var) : level(level), trange(trange), var(var) { } }; struct CursorData : public impl::CursorData { dballe::DBStation station; Datetime datetime; std::vector rows; std::vector::const_iterator cur; bool at_start = true; CursorData(const impl::Message& msg, bool merged=false) { station.report = msg.get_report(); station.coords = msg.get_coords(); station.ident = msg.get_ident(); datetime = msg.get_datetime(); for (const auto& ctx: msg.data) for (Values::const_iterator cur = ctx.values.begin(); cur != ctx.values.end(); ++cur) rows.emplace_back(ctx.level, ctx.trange, cur); if (merged) for (Values::const_iterator cur = msg.station_data.begin(); cur != msg.station_data.end(); ++cur) if (WR_VAR_X((*cur)->code()) < 4 || WR_VAR_X((*cur)->code()) > 6) rows.emplace_back(cur); } ~CursorData(); bool has_value() const override { return !at_start && cur != rows.end(); } int remaining() const override { if (at_start) return rows.size(); return rows.end() - cur; } bool next() override { if (at_start) { at_start = false; cur = rows.begin(); return true; } else if (cur == rows.end()) { return false; } else { ++cur; return cur != rows.end(); } } void discard() override { at_start = false; cur = rows.end(); } void enq(Enq& enq) const override; DBStation get_station() const override { return station; } wreport::Varcode get_varcode() const override { return (*(cur->var))->code(); } wreport::Var get_var() const override { return **(cur->var); } Level get_level() const override { return cur->level; } Trange get_trange() const override { return cur->trange; } Datetime get_datetime() const override { return datetime; } /// Downcast a unique_ptr pointer inline static std::unique_ptr downcast(std::unique_ptr c) { CursorData* res = dynamic_cast(c.get()); if (!res) throw std::runtime_error("Attempted to downcast the wrong kind of cursor"); c.release(); return std::unique_ptr(res); } }; } } } #endif dballe-8.6/dballe/msg/wr_import-test.cc0000644000175000017500000004637713554564112015101 00000000000000#include "tests.h" #include "wr_codec.h" #include "msg.h" #include "context.h" #include #include #include using namespace wreport; using namespace dballe; using namespace dballe::tests; using namespace std; namespace { #define IS(field, val) do { \ WREPORT_TEST_INFO(locinfo); \ locinfo() << #field; \ const Var* var = msg.get_##field##_var(); \ wassert(actual(var).istrue()); \ wassert(actual(*var) == val); \ } while (0) #define IS2(code, lev, tr, val) do { \ WREPORT_TEST_INFO(locinfo); \ locinfo() << #code #lev #tr; \ const Var* var = msg.get(lev, tr, code); \ wassert(actual(var).istrue()); \ wassert(actual(*var) == val); \ } while (0) #define UN(field) do { \ const Var* var = msg.get_##field##_var(); \ if (var != 0) \ wassert(actual(*var).isunset()); \ } while (0) class Tests : public TestCase { using TestCase::TestCase; void add_bufr_method(const std::string& fname, std::function m) { add_method("prec_" + fname, [=]() { std::string pathname = "bufr/" + fname; impl::ImporterOptions opts; opts.simplified = false; impl::Messages msgs = wcallchecked(read_msgs(pathname.c_str(), Encoding::BUFR, opts)); m(msgs); }); } void add_bufr_simplified_method(const std::string& fname, std::function m) { add_method("simp_" + fname, [=]() { std::string pathname = "bufr/" + fname; impl::ImporterOptions opts; opts.simplified = true; impl::Messages msgs = read_msgs(pathname.c_str(), Encoding::BUFR, opts); m(msgs); }); } void add_crex_method(const std::string& fname, std::function m) { add_method(fname, [=]() { std::string pathname = "crex/" + fname; impl::Messages msgs = read_msgs(pathname.c_str(), Encoding::CREX); m(msgs); }); } void register_tests() override { // Test plain import of all our BUFR test files add_method("all_bufr", []() { // note: These were blacklisted: // "bufr/obs3-3.1.bufr", // "bufr/obs3-56.2.bufr", // "bufr/test-buoy1.bufr", // "bufr/test-soil1.bufr", const char** files = dballe::tests::bufr_files; for (int i = 0; files[i] != NULL; i++) { try { impl::Messages msgs = read_msgs(files[i], Encoding::BUFR); wassert(actual(msgs.size()) > 0); } catch (std::exception& e) { cerr << "Failing bulletin:"; try { BinaryMessage raw = read_rawmsg(files[i], Encoding::BUFR); unique_ptr bulletin(BufrBulletin::decode(raw.data)); bulletin->print(stderr); } catch (std::exception& e1) { cerr << "Cannot display failing bulletin: " << e1.what() << endl; } throw TestFailed(string("[") + files[i] + "] " + e.what()); } } }); // Test plain import of all our CREX test files add_method("all_crex", []() { const char** files = dballe::tests::crex_files; for (int i = 0; files[i] != NULL; i++) { try { impl::Messages msgs = read_msgs(files[i], Encoding::CREX); wassert(actual(msgs.size()) > 0); } catch (std::exception& e) { cerr << "Failing bulletin:"; try { BinaryMessage raw = read_rawmsg(files[i], Encoding::CREX); unique_ptr bulletin(CrexBulletin::decode(raw.data)); bulletin->print(stderr); } catch (std::exception& e1) { cerr << "Cannot display failing bulletin: " << e1.what() << endl; } throw TestFailed(string("[") + files[i] + "] " + e.what()); } } }); add_crex_method("test-synop0.crex", [](const impl::Messages& msgs) { const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SYNOP); IS(block, 10); IS(station, 837); IS(st_type, 1); wassert(actual(msg.get_datetime()) == Datetime(2004, 11, 30, 12, 0)); IS(latitude, 48.22); IS(longitude, 9.92); IS(height_station, 550.0); UN(height_baro); IS(press, 94340.0); IS(press_msl, 100940.0); IS(press_tend, 7.0); IS(wind_dir, 80.0); IS(wind_speed, 6.0); IS(temp_2m, 276.15); IS(dewpoint_2m, 273.85); UN(humidity); IS(visibility, 5000.0); IS(pres_wtr, 10); IS(past_wtr1_6h, 2); IS(past_wtr2_6h, 2); IS(cloud_n, 100); IS(cloud_nh, 8); IS(cloud_hh, 450.0); IS(cloud_cl, 35); IS(cloud_cm, 61); IS(cloud_ch, 60); IS(cloud_n1, 8); IS(cloud_c1, 6); IS(cloud_h1, 350.0); UN(cloud_n2); UN(cloud_c2); UN(cloud_h2); UN(cloud_n3); UN(cloud_c3); UN(cloud_h3); UN(cloud_n4); UN(cloud_c4); UN(cloud_h4); UN(tot_prec24); UN(tot_snow); }); add_bufr_simplified_method("obs0-1.22.bufr", [](const impl::Messages& msgs) { const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SYNOP); IS(block, 60); IS(station, 150); IS(st_type, 1); wassert(actual(msg.get_datetime()) == Datetime(2004, 11, 30, 12, 0)); IS(latitude, 33.88); IS(longitude, -5.53); IS(height_station, 560.0); UN(height_baro); IS(press, 94190.0); IS(press_msl, 100540.0); IS(press_3h, -180.0); IS(press_tend, 8.0); IS(wind_dir, 80.0); IS(wind_speed, 4.0); IS(temp_2m, 289.2); IS(dewpoint_2m, 285.7); UN(humidity); IS(visibility, 8000.0); IS(pres_wtr, 2); IS(past_wtr1_6h, 6); IS(past_wtr2_6h, 2); IS(cloud_n, 100); IS(cloud_nh, 8); IS(cloud_hh, 250.0); IS(cloud_cl, 39); IS(cloud_cm, 61); IS(cloud_ch, 60); IS(cloud_n1, 2); IS(cloud_c1, 8); IS(cloud_h1, 320.0); IS(cloud_n2, 5); IS(cloud_c2, 8); IS(cloud_h2, 620.0); IS(cloud_n3, 2); IS(cloud_c3, 9); IS(cloud_h3, 920.0); UN(cloud_n4); UN(cloud_c4); UN(cloud_h4); IS(tot_prec12, 0.5); UN(tot_snow); }); add_bufr_simplified_method("synop-cloudbelow.bufr", [](const impl::Messages& msgs) { const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SYNOP); // msg.print(stderr); IS(block, 11); IS(station, 406); IS(st_type, 1); wassert(actual(msg.get_datetime()) == Datetime(2009, 12, 3, 15, 0)); IS(latitude, 50.07361); IS(longitude, 12.40333); IS(height_station, 483.0); IS(height_baro, 490.0); IS(press, 95090.0); IS(press_msl, 101060.0); IS(press_3h, -110.0); IS(press_tend, 6.0); IS(wind_dir, 0.0); IS(wind_speed, 1.0); IS(temp_2m, 273.05); IS(dewpoint_2m, 271.35); IS(humidity, 88.0); IS(visibility, 14000.0); IS(pres_wtr, 508); IS2(WR_VAR(0, 20, 4), Level(1), Trange(205, 0, 10800), 10); // past_wtr1 IS2(WR_VAR(0, 20, 5), Level(1), Trange(205, 0, 10800), 10); // past_wtr2 IS(cloud_n, 38); IS(cloud_nh, 0); IS(cloud_hh, 6000.0); IS(cloud_cl, 30); IS(cloud_cm, 20); IS(cloud_ch, 12); IS(cloud_n1, 3); IS(cloud_c1, 0); IS(cloud_h1, 6000.0); UN(cloud_n2); UN(cloud_c2); UN(cloud_h2); UN(cloud_n3); UN(cloud_c3); UN(cloud_h3); UN(cloud_n4); UN(cloud_c4); UN(cloud_h4); UN(tot_prec24); UN(tot_snow); }); add_bufr_method("synop-cloudbelow.bufr", [](const impl::Messages& msgs) { const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SYNOP); // msg.print(stderr); IS(block, 11); IS(station, 406); IS(st_type, 1); wassert(actual(msg.get_datetime()) == Datetime(2009, 12, 3, 15, 0)); IS(latitude, 50.07361); IS(longitude, 12.40333); IS(height_station, 483.0); IS(height_baro, 490.0); IS2(WR_VAR(0, 10, 4), Level(102, 490000), Trange::instant(), 95090.0); // press IS2(WR_VAR(0, 10, 51), Level(102, 490000), Trange::instant(), 101060.0); // press_msl IS2(WR_VAR(0, 10, 63), Level(102, 490000), Trange(205, 0,10800), 6.0); // press_tend IS2(WR_VAR(0, 10, 60), Level(102, 490000), Trange(4, 0, 10800), -110.0); // press_3h IS2(WR_VAR(0, 11, 1), Level(103, 10000), Trange(200, 0, 600), 0.0); // wind_dir IS2(WR_VAR(0, 11, 2), Level(103, 10000), Trange(200, 0, 600), 1.0); // wind_speed IS2(WR_VAR(0, 12, 101), Level(103, 2050), Trange::instant(), 273.05); // temp_2m IS2(WR_VAR(0, 12, 103), Level(103, 2050), Trange::instant(), 271.35); // dewpoint_2m IS2(WR_VAR(0, 13, 3), Level(103, 2050), Trange::instant(), 88.0); // humidity IS2(WR_VAR(0, 20, 1), Level(103, 8000), Trange::instant(), 14000.0); // visibility IS(pres_wtr, 508); IS2(WR_VAR(0, 20, 4), Level(1), Trange(205, 0, 10800), 10); // past_wtr1 IS2(WR_VAR(0, 20, 5), Level(1), Trange(205, 0, 10800), 10); // past_wtr2 IS(cloud_n, 38); IS(cloud_nh, 0); IS(cloud_hh, 6000.0); IS(cloud_cl, 30); IS(cloud_cm, 20); IS(cloud_ch, 12); IS(cloud_n1, 3); IS(cloud_c1, 0); IS(cloud_h1, 6000.0); UN(cloud_n2); UN(cloud_c2); UN(cloud_h2); UN(cloud_n3); UN(cloud_c3); UN(cloud_h3); UN(cloud_n4); UN(cloud_c4); UN(cloud_h4); UN(tot_prec24); UN(tot_snow); }); add_bufr_simplified_method("temp-2-255.bufr", [](const impl::Messages& msgs) { const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); // No negative pressure layers please wassert(actual(msg.find_context(Level(100, -1), Trange::instant())).isfalse()); }); add_bufr_simplified_method("synop-longname.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 7u); const impl::Message& msg = impl::Message::downcast(*msgs[2]); wassert(actual(msg.type) == MessageType::SYNOP); // Check that the long station name has been correctly truncated on import const Var* var = msg.get_st_name_var(); wassert(actual(var).istrue()); wassert(actual(string(var->enqc())) == "Budapest Pestszentlorinc-kulteru"); }); add_bufr_simplified_method("temp-bad1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); add_bufr_simplified_method("temp-bad2.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); add_bufr_simplified_method("temp-bad3.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); add_bufr_simplified_method("temp-bad4.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); // ECWMF AIREP add_bufr_simplified_method("obs4-142.1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::AIREP); IS(ident, "ACA872"); }); // ECWMF AMDAR add_bufr_simplified_method("obs4-144.4.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::AMDAR); IS(ident, "EU4444"); }); // ECWMF ACARS add_bufr_simplified_method("obs4-145.4.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::ACARS); IS(ident, "JBNYR3RA"); }); // WMO ACARS add_bufr_simplified_method("gts-acars1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::ACARS); IS(ident, "EU5331"); }); // WMO ACARS add_bufr_simplified_method("gts-acars2.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::ACARS); IS(ident, "FJCYR4RA"); }); // WMO ACARS UK add_bufr_simplified_method("gts-acars-uk1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); // This contains the same data as an AMDAR and has undefined subtype and // localsubtype, so it gets identified as an AMDAR wassert(actual(msg.type) == MessageType::AMDAR); IS(ident, "EU3375"); }); // WMO ACARS US add_bufr_simplified_method("gts-acars-us1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::ACARS); IS(ident, "FJCYR4RA"); }); // BUFR that has a variable that goes out of range when converted to local B // table add_method("outofrange", []() { try { // Read and interpretate the message BinaryMessage raw = read_rawmsg("bufr/interpreted-range.bufr", Encoding::BUFR); std::unique_ptr importer = Importer::create(Encoding::BUFR); impl::Messages msgs = importer->from_binary(raw); throw TestFailed("error_domain was not thrown"); } catch (wreport::error_domain& e) { //cerr << e.code() << "--" << e.what() << endl; } { wreport::options::LocalOverride o(wreport::options::var_silent_domain_errors, true); impl::Messages msgs = read_msgs("bufr/interpreted-range.bufr", Encoding::BUFR); wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SHIP); IS(ident, "DBBC"); } }); // WMO PILOT, with pressure levels add_bufr_simplified_method("pilot-gts2.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::PILOT); }); // WMO PILOT, with pressure levels add_bufr_simplified_method("temp-tsig-2.bufr", [](const impl::Messages& msgs) { // FIXME: this still fails wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); // WMO pilot pressure add_bufr_simplified_method("pilot-gts3.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::PILOT); }); // WMO pilot geopotential add_bufr_simplified_method("pilot-gts4.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::PILOT); }); add_bufr_simplified_method("vad.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); // Wind profiler add_bufr_simplified_method("temp-windprof1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::TEMP); }); // Precise import add_bufr_method("gts-synop-linate.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); const Var* v = msg.get(Level(103, 2000), Trange(3, 0, 43200), WR_VAR(0, 12, 101)); wassert(actual(v).istrue()); wassert(actual(v->enqd()) == 284.75); }); // Soil temperature (see https://github.com/ARPA-SIMC/dballe/issues/41 ) add_bufr_simplified_method("test-soil1.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::SYNOP); IS2(WR_VAR(0, 12, 30), Level(106, 50), Trange::instant(), 288.5); IS2(WR_VAR(0, 12, 30), Level(106, 100), Trange::instant(), 289.4); IS2(WR_VAR(0, 12, 30), Level(106, 200), Trange::instant(), 288.6); IS2(WR_VAR(0, 12, 30), Level(106, 500), Trange::instant(), 288.8); IS2(WR_VAR(0, 12, 30), Level(106, 1000), Trange::instant(), 288.4); }); // Truncated UTF8 in station name add_bufr_simplified_method("truncated-unicode.bufr", [](const impl::Messages& msgs) { wassert(actual(msgs.size()) == 1u); const impl::Message& msg = impl::Message::downcast(*msgs[0]); wassert(actual(msg.type) == MessageType::GENERIC); IS2(WR_VAR(0, 1, 19), Level(), Trange(), "Rocca San Giovanni, C.da Vallev\xc3"); IS2(WR_VAR(0, 13, 11), Level(1), Trange(1, 0, 21600), 0.0); }); } } test("msg_wr_import"); } dballe-8.6/dballe/msg/bulletin.h0000644000175000017500000000116613554564112013545 00000000000000#ifndef DBALLE_MSG_BULLETIN_H #define DBALLE_MSG_BULLETIN_H #include namespace wreport { struct Bulletin; } namespace dballe { namespace msg { /** * Write bulletins in CSV format to an output stream. * * Headers will only be written for the first bulletin, and will not be written * output_bulletin is never called. * * The output stream will be left open when the class is destroyed. */ class BulletinCSVWriter { protected: bool first = true; FILE* out; public: BulletinCSVWriter(FILE* out); ~BulletinCSVWriter(); void output_bulletin(const wreport::Bulletin& bulletin); }; } } #endif dballe-8.6/dballe/msg/cursor-test.cc0000644000175000017500000000146513554573614014370 00000000000000#include "dballe/msg/tests.h" #include "cursor.h" #include using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override; } tests("msg_cursor"); void Tests::register_tests() { add_method("issue160", []() { auto msgs = read_msgs("bufr/issue160.bufr", Encoding::BUFR); std::shared_ptr msg(impl::Message::downcast(msgs[0])); const Values& station_values = msg->find_station_context(); wassert(actual(station_values.size()) == 12); core::Query query; auto cur = msg->query_station_data(query); wassert(actual(cur->remaining()) == 12); unsigned iterations = 0; while (cur->next()) ++iterations; wassert(actual(iterations) == 12); }); } } dballe-8.6/dballe/msg/cursor-access.in.cc0000644000175000017500000000767613554564112015262 00000000000000#include "cursor.h" #include using namespace wreport; namespace dballe { namespace impl { namespace msg { void CursorStation::enq(impl::Enq& enq) const { if (enq.search_b_values(station_values)) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": return; case "rep_memo": enq.set_string(station.report); case "report": enq.set_string(station.report); case "ana_id": enq.set_dballe_int(station.id); case "mobile": enq.set_bool(!station.ident.is_missing()); case "ident": enq.set_ident(station.ident); case "lat": enq.set_lat(station.coords.lat); case "lon": enq.set_lon(station.coords.lon); case "coords": enq.set_coords(station.coords); case "station": enq.set_station(station); default: enq.search_alias_values(station_values); } } void CursorStationData::enq(impl::Enq& enq) const { if (enq.search_b_value(*cur)) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": return; case "rep_memo": enq.set_string(station.report); case "report": enq.set_string(station.report); case "ana_id": enq.set_dballe_int(station.id); case "mobile": enq.set_bool(!station.ident.is_missing()); case "ident": enq.set_ident(station.ident); case "lat": enq.set_lat(station.coords.lat); case "lon": enq.set_lon(station.coords.lon); case "coords": enq.set_coords(station.coords); case "station": enq.set_station(station); case "var": enq.set_varcode(cur->code()); case "variable": enq.set_var(cur->get()); case "attrs": enq.set_attrs(cur->get()); case "context_id": return; default: enq.search_alias_value(*cur); } } void CursorData::enq(impl::Enq& enq) const { if (enq.search_b_value(*(cur->var))) return; const auto key = enq.key; const auto len = enq.len; switch (key) { // mklookup case "priority": return; case "rep_memo": enq.set_string(station.report); case "report": enq.set_string(station.report); case "ana_id": enq.set_dballe_int(station.id); case "mobile": enq.set_bool(!station.ident.is_missing()); case "ident": enq.set_ident(station.ident); case "lat": enq.set_lat(station.coords.lat); case "lon": enq.set_lon(station.coords.lon); case "coords": enq.set_coords(station.coords); case "station": enq.set_station(station); case "datetime": enq.set_datetime(datetime); case "year": enq.set_int(datetime.year); case "month": enq.set_int(datetime.month); case "day": enq.set_int(datetime.day); case "hour": enq.set_int(datetime.hour); case "min": enq.set_int(datetime.minute); case "sec": enq.set_int(datetime.second); case "level": enq.set_level(cur->level); case "leveltype1": enq.set_dballe_int(cur->level.ltype1); case "l1": enq.set_dballe_int(cur->level.l1); case "leveltype2": enq.set_dballe_int(cur->level.ltype2); case "l2": enq.set_dballe_int(cur->level.l2); case "trange": enq.set_trange(cur->trange); case "pindicator": enq.set_dballe_int(cur->trange.pind); case "p1": enq.set_dballe_int(cur->trange.p1); case "p2": enq.set_dballe_int(cur->trange.p2); case "var": enq.set_varcode(cur->var->code()); case "variable": enq.set_var(cur->var->get()); case "attrs": enq.set_attrs(cur->var->get()); case "context_id": return; default: enq.search_alias_value(*(cur->var)); } } } } } dballe-8.6/dballe/msg/msg.cc0000644000175000017500000010040113554564112012643 00000000000000#include "msg.h" #include "context.h" #include "dballe/cursor.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/cursor.h" #include "dballe/core/var.h" #include "dballe/core/csv.h" #include #include #include #include #include #include #include #include using namespace wreport; using namespace std; namespace dballe { namespace impl { namespace msg { Contexts::const_iterator Contexts::find(const Level& level, const Trange& trange) const { /* Binary search */ if (m_contexts.empty()) return m_contexts.end(); const_iterator low = m_contexts.begin(), high = (m_contexts.end() - 1); while (low <= high) { const_iterator middle = low + (high - low) / 2; int cmp = middle->compare(level, trange); if (cmp > 0) high = middle - 1; else if (cmp < 0) low = middle + 1; else return middle; } return m_contexts.end(); } Contexts::iterator Contexts::find(const Level& level, const Trange& trange) { /* Binary search */ if (m_contexts.empty()) return m_contexts.end(); iterator low = m_contexts.begin(), high = (m_contexts.end() - 1); while (low <= high) { iterator middle = low + (high - low) / 2; int cmp = middle->compare(level, trange); if (cmp > 0) high = middle - 1; else if (cmp < 0) low = middle + 1; else return middle; } return m_contexts.end(); } Contexts::iterator Contexts::insert_new(const Level& level, const Trange& trange) { // Enlarge the buffer m_contexts.emplace_back(level, trange); // Insertionsort iterator pos; for (pos = m_contexts.end() - 1; pos > m_contexts.begin(); --pos) { if ((pos - 1)->compare(*pos) > 0) std::swap(*pos, *(pos - 1)); else break; } return pos; } Contexts::iterator Contexts::obtain(const Level& level, const Trange& trange) { iterator pos = find(level, trange); if (pos != end()) return pos; return insert_new(level, trange); } bool Contexts::drop(const Level& level, const Trange& trange) { iterator pos = find(level, trange); if (pos == end()) return false; m_contexts.erase(pos); return true; } Messages messages_from_csv(CSVReader& in) { Messages res; string old_rep; bool first = true; while (true) { // Seek to beginning, skipping empty lines if (!in.move_to_data()) return res; if (in.cols.size() != 13) error_consistency::throwf("cannot parse CSV line has %zd fields instead of 13", in.cols.size()); if (first) { // If we are the first run, initialse old_* markers with the contents of this line old_rep = in.cols[2]; first = false; } else if (old_rep != in.cols[2]) // If Report changes, we are done break; auto msg = make_shared(); bool has_next = msg->from_csv(in); res.emplace_back(std::move(msg)); if (!has_next) break; } return res; } void messages_to_csv(const Messages& msgs, CSVWriter& out) { for (const auto& i: msgs) impl::Message::downcast(i)->to_csv(out); } unsigned messages_diff(const Messages& msgs1, const Messages& msgs2) { unsigned diffs = 0; if (msgs1.size() != msgs2.size()) { notes::logf("the message groups contain a different number of messages (first is %zd, second is %zd)\n", msgs1.size(), msgs2.size()); ++diffs; } size_t count = min(msgs1.size(), msgs2.size()); for (size_t i = 0; i < count; ++i) diffs += msgs1[i]->diff(*msgs2[i]); return diffs; } void messages_print(const Messages& msgs, FILE* out) { for (unsigned i = 0; i < msgs.size(); ++i) { fprintf(out, "Subset %d:\n", i); msgs[i]->print(out); } } } const Message& Message::downcast(const dballe::Message& o) { const Message* ptr = dynamic_cast(&o); if (!ptr) throw error_consistency("Message given is not an impl::Message"); return *ptr; } Message& Message::downcast(dballe::Message& o) { Message* ptr = dynamic_cast(&o); if (!ptr) throw error_consistency("Message given is not an impl::Message"); return *ptr; } std::shared_ptr Message::downcast(std::shared_ptr o) { auto ptr = dynamic_pointer_cast(o); if (!ptr) throw error_consistency("Message given is not a Message"); return ptr; } std::unique_ptr Message::clone() const { return unique_ptr(new Message(*this)); } Datetime Message::get_datetime() const { int ye = MISSING_INT, mo=MISSING_INT, da=MISSING_INT, ho=MISSING_INT, mi=MISSING_INT, se=MISSING_INT; if (const Var* v = station_data.maybe_var(sc::year.code)) ye = v->enqi(); if (const Var* v = station_data.maybe_var(sc::month.code)) mo = v->enqi(); if (const Var* v = station_data.maybe_var(sc::day.code)) da = v->enqi(); if (const Var* v = station_data.maybe_var(sc::hour.code)) ho = v->enqi(); if (const Var* v = station_data.maybe_var(sc::minute.code)) mi = v->enqi(); if (const Var* v = station_data.maybe_var(sc::second.code)) se = v->enqi(); if (ye == MISSING_INT) return Datetime(); if (mo == MISSING_INT) throw error_consistency("no month information found in message"); if (da == MISSING_INT) throw error_consistency("no day information found in message"); if (ho == MISSING_INT) throw error_consistency("no hour information found in message"); if (mi == MISSING_INT) throw error_consistency("no minute information found in message"); if (se == MISSING_INT) se = 0; // Accept an hour of 24:00:00 and move it to 00:00:00 of the following // day Datetime::normalise_h24(ye, mo, da, ho, mi, se); return Datetime(ye, mo, da, ho, mi, se); } Coords Message::get_coords() const { const Var* lat = station_data.maybe_var(sc::latitude.code); const Var* lon = station_data.maybe_var(sc::longitude.code); if (lat && lon) return Coords(lat->enqd(), lon->enqd()); else return Coords(); } Ident Message::get_ident() const { const Var* ident = station_data.maybe_var(sc::ident.code); if (ident) return Ident(ident->enqc()); else return Ident(); } std::string Message::get_report() const { // Postprocess extracting rep_memo information const Var* rep_memo = station_data.maybe_var(sc::rep_memo.code); if (rep_memo) return rep_memo->enqc(); else return repmemo_from_type(type); } void Message::clear() { type = MessageType::GENERIC; station_data.clear(); data.clear(); } const msg::Context* Message::find_context(const Level& lev, const Trange& tr) const { if (lev.is_missing() && tr.is_missing()) throw std::runtime_error("find_contexts called for station level, but this is no longer supported"); auto i = data.find(lev, tr); if (i == data.end()) return nullptr; return &*i; } const Values& Message::find_station_context() const { return station_data; } msg::Context* Message::edit_context(const Level& lev, const Trange& tr) { if (lev.is_missing() && tr.is_missing()) throw std::runtime_error("find_contexts called for station level, but this is no longer supported"); auto i = data.find(lev, tr); if (i == data.end()) return nullptr; return &*i; } msg::Context& Message::obtain_context(const Level& lev, const Trange& tr) { if (lev.is_missing() && tr.is_missing()) throw std::runtime_error("find_contexts called for station level, but this is no longer supported"); auto i = data.obtain(lev, tr); return *i; } bool Message::remove_context(const Level& lev, const Trange& tr) { return data.drop(lev, tr); } const Var* Message::get_impl(const Level& lev, const Trange& tr, Varcode code) const { if (lev.is_missing() && tr.is_missing()) return station_data.maybe_var(code); auto ctx = data.find(lev, tr); if (ctx == data.end()) return nullptr; return ctx->values.maybe_var(code); } bool Message::foreach_var(std::function dest) const { for (const auto& var: station_data) if (!dest(Level(), Trange(), *var)) return false; for (const auto& ctx: data) for (const auto& var: ctx.values) if (!dest(ctx.level, ctx.trange, *var)) return false; return true; } wreport::Var* Message::edit(wreport::Varcode code, const Level& lev, const Trange& tr) { if (lev.is_missing() && tr.is_missing()) throw std::runtime_error("find_contexts called for station level, but this is no longer supported"); auto ctx = data.find(lev, tr); if (ctx == data.end()) return nullptr; return ctx->values.maybe_var(code); } const Var* Message::get(const Shortcut& shortcut) const { if (shortcut.station_data) return station_data.maybe_var(shortcut.code); return get(shortcut.level, shortcut.trange, shortcut.code); } std::unique_ptr Message::query_stations(const Query& query) const { return std::unique_ptr(new msg::CursorStation(*this)); } std::unique_ptr Message::query_station_data(const Query& query) const { return std::unique_ptr(new msg::CursorStationData(*this)); } std::unique_ptr Message::query_data(const Query& query) const { return std::unique_ptr(new msg::CursorData(*this)); } std::unique_ptr Message::query_station_and_data(const Query& query) const { return std::unique_ptr(new msg::CursorData(*this, true)); } namespace { struct VarContext { const impl::Message& msg; // Extract datetime, lat, lon const Var* lat; const Var* lon; const Var* memo; const char* rep_memo; VarContext(const impl::Message& m) : msg(m) { // Extract datetime, lat, lon lat = m.station_data.maybe_var(sc::latitude.code); lon = m.station_data.maybe_var(sc::longitude.code); memo = m.station_data.maybe_var(sc::rep_memo.code); if (memo) rep_memo = memo->enqc(); else rep_memo = impl::Message::repmemo_from_type(m.type); } void print(CSVWriter& out, const Level& level, const Trange& trange) { // Longitude if (lon) out.add_var_value_formatted(*lon); else out.add_value_empty(); // Latitude if (lat) out.add_var_value_formatted(*lat); else out.add_value_empty(); // Report type out.add_value(rep_memo); if (level != Level()) { // Datetime msg.get_datetime().to_csv_iso8601(out, ' '); // Level level.to_csv(out); // Time range trange.to_csv(out); } else { for (int i = 0; i < 8; ++i) out.add_value_empty(); } } }; } void Message::to_csv(CSVWriter& out) const { VarContext vc(*this); for (const auto& val: station_data) { vc.print(out, Level(), Trange()); out.add_value(val->code()); // B code out.add_var_value_formatted(*val); out.flush_row(); // Add attribute columns for (const Var* a = val->next_attr(); a != NULL; a = a->next_attr()) { vc.print(out, Level(), Trange()); out.add_value(varcode_format(val->code()) + "." + varcode_format(a->code())); // B code out.add_var_value_formatted(*a); out.flush_row(); } } for (const auto& ctx: data) { for (const auto& val: ctx.values) { const Var& v = *val; vc.print(out, ctx.level, ctx.trange); out.add_value(v.code()); // B code out.add_var_value_formatted(v); out.flush_row(); // Add attribute columns for (const Var* a = v.next_attr(); a != NULL; a = a->next_attr()) { vc.print(out, ctx.level, ctx.trange); out.add_value(varcode_format(v.code()) + "." + varcode_format(a->code())); // B code out.add_var_value_formatted(*a); out.flush_row(); } } } } void Message::csv_header(CSVWriter& out) { out.add_value("longitude"); out.add_value("latitude"); out.add_value("report"); out.add_value("date"); out.add_value("level"); out.add_value("l"); out.add_value("level"); out.add_value("l"); out.add_value("time rang"); out.add_value("p"); out.add_value("p"); out.add_value("varcod"); out.add_value("value"); out.flush_row(); } namespace { // Convert a string to an integer value, returning MISSING_INT if the string is // empty or "-" int str_to_int(const std::string& str) { if (str.empty() || str == "-") return MISSING_INT; else return stoi(str); } } bool Message::from_csv(CSVReader& in) { // Seek to beginning, skipping empty lines if (!in.move_to_data()) return false; string old_lat, old_lon, old_rep, old_date; bool first = true; while (true) { // If there are empty lines, use them as separators if (in.cols.empty()) break; if (in.cols.size() != 13) error_consistency::throwf("cannot parse CSV line has %zd fields instead of 13", in.cols.size()); if (first) { // If we are the first run, initialse old_* markers with the contents of this line old_lon = in.cols[0]; old_lat = in.cols[1]; old_rep = in.cols[2]; old_date = in.cols[3]; set_latitude(strtod(old_lat.c_str(), NULL)); set_longitude(strtod(old_lon.c_str(), NULL)); set_rep_memo(old_rep.c_str()); if (!old_date.empty()) set_datetime(Datetime::from_iso8601(old_date.c_str())); type = type_from_repmemo(old_rep.c_str()); first = false; } else if (old_lon != in.cols[0] || old_lat != in.cols[1] || old_rep != in.cols[2]) { // If Longitude, Latitude or Report change, we are done break; } else if (old_date != in.cols[3]) { // In case of Date differences, we need to deal with station // information for which the date is left empty if (old_date.empty()) { // previous lines were station information, next line is data old_date = in.cols[3]; set_datetime(Datetime::from_iso8601(old_date.c_str())); } else if (in.cols[3].empty()) // previous lines were data, next line is station information ; // Keep the old date else // The date has changed, we are done. break; } // 0 1 2 3 4 5 6 7 8 9 10 11 12 // out << "Longitude,Latitude,Report,Date,Level1,L1,Level2,L2,Time range,P1,P2,Varcode,Value" << endl; // Acquire the data Level lev(str_to_int(in.cols[4]), str_to_int(in.cols[5]), str_to_int(in.cols[6]), str_to_int(in.cols[7])); if (in.cols[3].empty()) // If we have station info, set level accordingly lev = Level(); Trange tr(str_to_int(in.cols[8]), str_to_int(in.cols[9]), str_to_int(in.cols[10])); // Parse variable code if (in.cols[11].size() == 13) { // Bxxyyy.Bxxyyy: attribute Varcode vcode = varcode_parse(in.cols[11].substr(0, 6).c_str()); // Find master variable wreport::Var* var = edit(vcode, lev, tr); if (var == NULL) error_consistency::throwf("cannot find corresponding variable for attribute %s", in.cols[11].c_str()); Varcode acode = varcode_parse(in.cols[11].substr(7).c_str()); auto attr = newvar(acode); attr->setf(in.cols[12].c_str()); var->seta(move(attr)); } else if (in.cols[11].size() == 6) { // Bxxyyy: variable Varcode vcode = varcode_parse(in.cols[11].c_str()); unique_ptr var = newvar(vcode); var->setf(in.cols[12].c_str()); if (lev.is_missing() && tr.is_missing()) station_data.set(std::move(var)); else set(lev, tr, std::move(var)); } else error_consistency::throwf("cannot parse variable code %s", in.cols[11].c_str()); if (!in.next()) break; } return true; } void Message::print(FILE* out) const { fprintf(out, "%s message, ", format_message_type(type)); get_coords().print(out, ", "); auto ident = get_ident(); if (!ident.is_missing()) fprintf(out, "ident: %s, ", (const char*)ident); auto dt = get_datetime(); if (dt.is_missing()) fprintf(out, "dt: missing, "); else { fprintf(out, "dt: "); dt.print_iso8601(out, 'T', ", "); } if (data.empty()) fprintf(out, "(no data)\n"); else fprintf(out, "%zd contexts:\n", data.size() + 1); fprintf(out, "Level "); Level().print(out, "-", " tr "); Trange().print(out, "-", "\n"); station_data.print(out); switch (type) { case MessageType::PILOT: case MessageType::TEMP: case MessageType::TEMP_SHIP: { unsigned sounding_idx = 0; for (auto i = data.cbegin(); i != data.cend(); ++i) { if (const Var* vsig = i->find_vsig()) { int vs = vsig->enqi(); fprintf(out, "Sounding #%u (level %d -", ++sounding_idx, vs); if (vs & BUFR08042::MISSING) fprintf(out, " missing"); if (vs & BUFR08042::H2PRESS) fprintf(out, " h2press"); if (vs & BUFR08042::RESERVED) fprintf(out, " reserved"); if (vs & BUFR08042::REGIONAL) fprintf(out, " regional"); if (vs & BUFR08042::TOPWIND) fprintf(out, " topwind"); if (vs & BUFR08042::ENDMISSW) fprintf(out, " endmissw"); if (vs & BUFR08042::BEGMISSW) fprintf(out, " begmissw"); if (vs & BUFR08042::ENDMISSH) fprintf(out, " endmissh"); if (vs & BUFR08042::BEGMISSH) fprintf(out, " begmissh"); if (vs & BUFR08042::ENDMISST) fprintf(out, " endmisst"); if (vs & BUFR08042::BEGMISST) fprintf(out, " begmisst"); if (vs & BUFR08042::SIGWIND) fprintf(out, " sigwind"); if (vs & BUFR08042::SIGHUM) fprintf(out, " sighum"); if (vs & BUFR08042::SIGTEMP) fprintf(out, " sigtemp"); if (vs & BUFR08042::MAXWIND) fprintf(out, " maxwind"); if (vs & BUFR08042::TROPO) fprintf(out, " tropo"); if (vs & BUFR08042::STD) fprintf(out, " std"); if (vs & BUFR08042::SURFACE) fprintf(out, " surface"); fprintf(out, ") "); } i->print(out); } break; } default: for (const auto& ctx: data) ctx.print(out); break; } } static void context_summary(const msg::Context& c, ostream& out) { out << "c(" << c.level << ", " << c.trange << ")"; } static void station_data_summary(const Var& var, ostream& out) { out << "Station variable "; out << varcode_format(var.code()) << "[" << var.info()->desc << "]"; } unsigned Message::diff(const dballe::Message& o) const { const Message& msg = downcast(o); unsigned diffs = 0; if (type != msg.type) { notes::logf("the messages have different type (first is %s (%d), second is %s (%d))\n", format_message_type(type), static_cast(type), format_message_type(msg.type), static_cast(msg.type)); ++diffs; } // Compare station data auto v1 = station_data.cbegin(); auto v2 = msg.station_data.cbegin(); while (v1 != station_data.cend() && v2 != msg.station_data.cend()) { // Skip second=0 in station context if (v1->code() == WR_VAR(0, 4, 6) && (*v1)->enqi() == 0) ++v1; if (v2->code() == WR_VAR(0, 4, 6) && (*v2)->enqi() == 0) ++v2; if (v1 == station_data.end() || v2 == msg.station_data.end()) break; int cmp = (int)v1->code() - (int)v2->code(); if (cmp == 0) { diffs += (*v1)->diff(**v2); ++v1; ++v2; } else if (cmp < 0) { if (!(*v1)->isset()) { station_data_summary(**v1, notes::log()); notes::log() << " exists only in the first message" << endl; ++diffs; } ++v1; } else { if (!(*v2)->isset()) { station_data_summary(**v2, notes::log()); notes::log() << " exists only in the second message" << endl; ++diffs; } ++v2; } } while (v1 != station_data.end()) { station_data_summary(**v1, notes::log()); notes::log() << " exists only in the first message" << endl; ++v1; ++diffs; } while (v2 != msg.station_data.end()) { station_data_summary(**v2, notes::log()); notes::log() << " exists only in the second message" << endl; ++v2; ++diffs; } auto i1 = data.cbegin(); auto i2 = msg.data.cbegin(); while (i1 != data.cend() && i2 != msg.data.cend()) { int cmp = i1->compare(*i2); if (cmp == 0) { diffs += i1->diff(*i2); ++i1; ++i2; } else if (cmp < 0) { if (!i1->values.empty()) { notes::log() << "Context "; context_summary(*i1, notes::log()); notes::log() << " exists only in the first message" << endl; ++diffs; } ++i1; } else { if (!i2->values.empty()) { notes::log() << "Context "; context_summary(*i2, notes::log()); notes::log() << " exists only in the second message" << endl; ++diffs; } ++i2; } } while (i1 != data.end()) { notes::log() << "Context "; context_summary(*i1, notes::log()); notes::log() << " exists only in the first message" << endl; ++i1; ++diffs; } while (i2 != msg.data.end()) { notes::log() << "Context "; context_summary(*i2, notes::log()); notes::log() << " exists only in the second message" << endl; ++i2; ++diffs; } return diffs; } void Message::set(const Shortcut& shortcut, const wreport::Var& var) { if (shortcut.station_data) { if (shortcut.code == var.code()) station_data.set(var); else station_data.set(var_copy_without_unset_attrs(var, shortcut.code)); } else set(shortcut.level, shortcut.trange, shortcut.code, var); } void Message::set_impl(const Level& lev, const Trange& tr, std::unique_ptr var) { if (lev.is_missing() && tr.is_missing()) station_data.set(std::move(var)); else { msg::Context& ctx = obtain_context(lev, tr); ctx.values.set(std::move(var)); } } void Message::seti(const Level& lev, const Trange& tr, Varcode code, int val, int conf) { unique_ptr var(newvar(code, val)); if (conf != -1) var->seta(newvar(WR_VAR(0, 33, 7), conf)); if (lev.is_missing() && tr.is_missing()) station_data.set(std::move(var)); else set(lev, tr, std::move(var)); } void Message::setd(const Level& lev, const Trange& tr, Varcode code, double val, int conf) { unique_ptr var(newvar(code, val)); if (conf != -1) var->seta(newvar(WR_VAR(0, 33, 7), conf)); if (lev.is_missing() && tr.is_missing()) station_data.set(std::move(var)); else set(lev, tr, std::move(var)); } void Message::setc(const Level& lev, const Trange& tr, Varcode code, const char* val, int conf) { unique_ptr var(newvar(code, val)); if (conf != -1) var->seta(newvar(WR_VAR(0, 33, 7), conf)); if (lev.is_missing() && tr.is_missing()) station_data.set(std::move(var)); else set(lev, tr, std::move(var)); } MessageType Message::type_from_repmemo(const char* repmemo) { if (repmemo == NULL || repmemo[0] == 0) return MessageType::GENERIC; switch (tolower(repmemo[0])) { case 'a': if (strcasecmp(repmemo+1, "cars")==0) return MessageType::ACARS; if (strcasecmp(repmemo+1, "irep")==0) return MessageType::AIREP; if (strcasecmp(repmemo+1, "mdar")==0) return MessageType::AMDAR; break; case 'b': if (strcasecmp(repmemo+1, "uoy")==0) return MessageType::BUOY; break; case 'm': if (strcasecmp(repmemo+1, "etar")==0) return MessageType::METAR; break; case 'p': if (strcasecmp(repmemo+1, "ilot")==0) return MessageType::PILOT; if (strcasecmp(repmemo+1, "ollution")==0) return MessageType::POLLUTION; break; case 's': if (strcasecmp(repmemo+1, "atellite")==0) return MessageType::SAT; if (strcasecmp(repmemo+1, "hip")==0) return MessageType::SHIP; if (strcasecmp(repmemo+1, "ynop")==0) return MessageType::SYNOP; break; case 't': if (strcasecmp(repmemo+1, "emp")==0) return MessageType::TEMP; if (strcasecmp(repmemo+1, "empship")==0) return MessageType::TEMP_SHIP; break; } return MessageType::GENERIC; } const char* Message::repmemo_from_type(MessageType type) { switch (type) { case MessageType::SYNOP: return "synop"; case MessageType::METAR: return "metar"; case MessageType::SHIP: return "ship"; case MessageType::BUOY: return "buoy"; case MessageType::AIREP: return "airep"; case MessageType::AMDAR: return "amdar"; case MessageType::ACARS: return "acars"; case MessageType::PILOT: return "pilot"; case MessageType::TEMP: return "temp"; case MessageType::TEMP_SHIP: return "tempship"; case MessageType::SAT: return "satellite"; case MessageType::POLLUTION: return "pollution"; case MessageType::GENERIC: default: return "generic"; } } void Message::sounding_pack_levels() { msg::Contexts new_data; for (auto& ctx: data) { if (ctx.find_vsig()) // FIXME: shouldn't this also set significance bits in the output level? new_data.obtain(Level(ctx.level.ltype1, ctx.level.l1), ctx.trange)->values.merge(std::move(ctx.values)); else // If it is not a sounding level, just copy it new_data.obtain(ctx.level, ctx.trange)->values = std::move(ctx.values); } data = std::move(new_data); } void Message::set_datetime(const Datetime& dt) { set_year(dt.year); set_month(dt.month); set_day(dt.day); set_hour(dt.hour); set_minute(dt.minute); set_second(dt.second); } MatchedMsg::MatchedMsg(const Message& m) : m(m) { } MatchedMsg::~MatchedMsg() { } matcher::Result MatchedMsg::match_var_id(int value) const { for (const auto& val: m.station_data) if (const Var* a = val->enqa(WR_VAR(0, 33, 195))) if (a->enqi() == value) return matcher::MATCH_YES; for (const auto& ctx: m.data) for (const auto& val: ctx.values) if (const Var* a = val->enqa(WR_VAR(0, 33, 195))) if (a->enqi() == value) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMsg::match_station_id(int val) const { if (const wreport::Var* var = m.station_data.maybe_var(WR_VAR(0, 1, 192))) { return var->enqi() == val ? matcher::MATCH_YES : matcher::MATCH_NO; } else return matcher::MATCH_NA; } matcher::Result MatchedMsg::match_station_wmo(int block, int station) const { if (const wreport::Var* var = m.station_data.maybe_var(WR_VAR(0, 1, 1))) { // Match block if (var->enqi() != block) return matcher::MATCH_NO; // If station was not requested, we are done if (station == -1) return matcher::MATCH_YES; // Match station if (const wreport::Var* var = m.station_data.maybe_var(WR_VAR(0, 1, 2))) { if (var->enqi() != station) return matcher::MATCH_NO; return matcher::MATCH_YES; } } return matcher::MATCH_NA; } matcher::Result MatchedMsg::match_datetime(const DatetimeRange& range) const { Datetime dt = m.get_datetime(); if (dt.is_missing()) return matcher::MATCH_NA; return range.contains(dt) ? matcher::MATCH_YES : matcher::MATCH_NO; } matcher::Result MatchedMsg::match_coords(const LatRange& latrange, const LonRange& lonrange) const { Coords coords = m.get_coords(); if (coords.is_missing()) { matcher::Result r1 = latrange.is_missing() ? matcher::MATCH_YES : matcher::MATCH_NA; matcher::Result r2 = lonrange.is_missing() ? matcher::MATCH_YES : matcher::MATCH_NA; if (r1 == matcher::MATCH_YES && r2 == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result r1 = latrange.contains(coords.lat) ? matcher::MATCH_YES : matcher::MATCH_NO; matcher::Result r2 = lonrange.contains(coords.lon) ? matcher::MATCH_YES : matcher::MATCH_NO; if (r1 == matcher::MATCH_YES && r2 == matcher::MATCH_YES) return matcher::MATCH_YES; if (r1 == matcher::MATCH_NO || r2 == matcher::MATCH_NO) return matcher::MATCH_NO; return matcher::MATCH_NA; } matcher::Result MatchedMsg::match_rep_memo(const char* memo) const { if (const Var* var = m.station_data.maybe_var(sc::rep_memo.code)) { if (!var->isset()) return matcher::MATCH_NA; return strcmp(memo, var->enqc()) == 0 ? matcher::MATCH_YES : matcher::MATCH_NO; } else return matcher::MATCH_NA; } MatchedMessages::MatchedMessages(const Messages& m) : m(m) { } MatchedMessages::~MatchedMessages() { } matcher::Result MatchedMessages::match_var_id(int val) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_var_id(val) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMessages::match_station_id(int val) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_station_id(val) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMessages::match_station_wmo(int block, int station) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_station_wmo(block, station) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMessages::match_datetime(const DatetimeRange& range) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_datetime(range) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMessages::match_coords(const LatRange& latrange, const LonRange& lonrange) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_coords(latrange, lonrange) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } matcher::Result MatchedMessages::match_rep_memo(const char* memo) const { for (const auto& i: m) if (MatchedMsg(*impl::Message::downcast(i)).match_rep_memo(memo) == matcher::MATCH_YES) return matcher::MATCH_YES; return matcher::MATCH_NA; } } } dballe-8.6/dballe/msg/msg.h0000644000175000017500000002735113554564112012521 00000000000000#ifndef DBALLE_MSG_H #define DBALLE_MSG_H #include #include #include #include #include #include #include #include #include #include #include #include #include namespace dballe { struct CSVReader; struct CSVWriter; namespace impl { /// ImporterOptions with default constructor usable struct ImporterOptions : public dballe::ImporterOptions { ImporterOptions() = default; ImporterOptions(const std::string& s) : dballe::ImporterOptions(s) {} ImporterOptions(const ImporterOptions&) = default; ImporterOptions(ImporterOptions&&) = default; ImporterOptions& operator=(const ImporterOptions&) = default; ImporterOptions& operator=(ImporterOptions&&) = default; using dballe::ImporterOptions::operator==; using dballe::ImporterOptions::operator!=; }; /// ExporterOptions with default constructor usable struct ExporterOptions : public dballe::ExporterOptions { ExporterOptions() = default; ExporterOptions(const ExporterOptions&) = default; ExporterOptions(ExporterOptions&&) = default; ExporterOptions& operator=(const ExporterOptions&) = default; ExporterOptions& operator=(ExporterOptions&&) = default; using dballe::ExporterOptions::operator==; using dballe::ExporterOptions::operator!=; }; // Compatibility/shortcut from old Messages implementation to new vector of shared_ptr typedef std::vector> Messages; namespace msg { /** * Read data from a CSV input. * * Reading stops when Report changes. */ Messages messages_from_csv(CSVReader& in); /** * Output in CSV format */ void messages_to_csv(const Messages& msgs, CSVWriter& out); /** * Compute the differences between two Messages * * Details of the differences found will be formatted using the wreport * notes system (@see wreport/notes.h). * * @returns * The number of differences found */ unsigned messages_diff(const Messages& msgs1, const Messages& msgs2); /// Print all the contents of all the messages to an output stream void messages_print(const Messages& msgs, FILE* out); class Contexts { public: typedef std::vector::const_iterator const_iterator; typedef std::vector::iterator iterator; typedef std::vector::const_reverse_iterator const_reverse_iterator; typedef std::vector::reverse_iterator reverse_iterator; protected: std::vector m_contexts; iterator insert_new(const Level& level, const Trange& trange); public: Contexts() = default; Contexts(const Contexts&) = default; Contexts(Contexts&&) = default; Contexts& operator=(const Contexts&) = default; Contexts& operator=(Contexts&&) = default; const_iterator begin() const { return m_contexts.begin(); } const_iterator end() const { return m_contexts.end(); } iterator begin() { return m_contexts.begin(); } iterator end() { return m_contexts.end(); } const_reverse_iterator rbegin() const { return m_contexts.rbegin(); } const_reverse_iterator rend() const { return m_contexts.rend(); } const_iterator cbegin() const { return m_contexts.cbegin(); } const_iterator cend() const { return m_contexts.cend(); } const_iterator find(const Level& level, const Trange& trange) const; iterator find(const Level& level, const Trange& trange); iterator obtain(const Level& level, const Trange& trange); bool drop(const Level& level, const Trange& trange); size_t size() const { return m_contexts.size(); } bool empty() const { return m_contexts.empty(); } void clear() { return m_contexts.clear(); } void reserve(typename std::vector::size_type size) { m_contexts.reserve(size); } iterator erase(iterator pos) { return m_contexts.erase(pos); } // iterator erase(const_iterator pos) { return m_contexts.erase(pos); } }; } /** * Storage for related physical data */ class Message : public dballe::Message { protected: /** * Return the index of the given context, or -1 if it was not found */ int find_index(const Level& lev, const Trange& tr) const; const wreport::Var* get_impl(const Level& lev, const Trange& tr, wreport::Varcode code) const override; void set_impl(const Level& lev, const Trange& tr, std::unique_ptr var) override; void seti(const Level& lev, const Trange& tr, wreport::Varcode code, int val, int conf); void setd(const Level& lev, const Trange& tr, wreport::Varcode code, double val, int conf); void setc(const Level& lev, const Trange& tr, wreport::Varcode code, const char* val, int conf); public: /// Source of the data MessageType type = MessageType::GENERIC; Values station_data; msg::Contexts data; Message() = default; Message(const Message&) = default; Message(Message&&) = default; Message& operator=(const Message& m) = default; Message& operator=(Message&& m) = default; /** * Return a reference to \a o downcasted as an impl::Message. * * Throws an exception if \a o is not an impl::Message. */ static const Message& downcast(const dballe::Message& o); /** * Return a reference to \a o downcasted as an impl::Message. * * Throws an exception if \a o is not an impl::Message. */ static Message& downcast(dballe::Message& o); /** * Returns a pointer to \a o downcasted as an impl::Message. * * Throws an exception if \a o is not an impl::Message. */ static std::shared_ptr downcast(std::shared_ptr o); std::unique_ptr clone() const override; Datetime get_datetime() const override; Coords get_coords() const override; Ident get_ident() const override; std::string get_report() const override; MessageType get_type() const override { return type; } bool foreach_var(std::function) const override; void print(FILE* out) const override; unsigned diff(const dballe::Message& msg) const override; /// Reset the messages as if it was just created void clear(); using dballe::Message::get; using dballe::Message::set; /** * Find a datum given its shortcut * * @param shortcut * Shortcut of the value to set. * @return * The value found, or nullptr if it was not found. */ const wreport::Var* get(const Shortcut& shortcut) const; /** * Add or replace a value * * @param shortcut * Shortcut ID of the value to set * @param var * The Var with the value to set */ void set(const Shortcut& shortcut, const wreport::Var& var); /** * Shortcut to set year...second variables in a single call */ void set_datetime(const Datetime& dt); /** * Remove a context from the message * * @return true if the context was removed, false if it did not exist */ bool remove_context(const Level& lev, const Trange& tr); /** * Find a msg::Context given its description * * @param lev * The Level to query * @param tr * The Trange to query * @return * The context found, or NULL if it was not found. */ const msg::Context* find_context(const Level& lev, const Trange& tr) const; /** * Find the station info context * * @return * The context found, or NULL if it was not found. */ const Values& find_station_context() const; /** * Find a msg::Context given its description * * @param lev * The Level to query * @param tr * The Trange to query * @return * The context found, or NULL if it was not found. */ msg::Context* edit_context(const Level& lev, const Trange& tr); /** * Find a msg::Context given its description, creating it if it does not * exist * * @param lev * The Level to query * @param tr * The Trange to query * @return * The context found */ msg::Context& obtain_context(const Level& lev, const Trange& tr); /** * Find a variable given its description * * @param code * The wreport::Varcode of the variable to query. * @param lev * The Level to query * @param tr * The Trange to query * @return * The variable found, or NULL if it was not found. */ wreport::Var* edit(wreport::Varcode code, const Level& lev, const Trange& tr); #if 0 /** * Remove a variable given its description * * @param code * The wreport::Varcode of the variable to query. * @param lev * The Level to query * @param tr * The Trange to query * @returns * True if the variable was removed, false if it was not found. */ bool remove(wreport::Varcode code, const Level& lev, const Trange& tr); #endif /** * Remove the sounding significance from the level descriptions and pack * together the data at the same pressure level. * * This is used to postprocess data after decoding, where the l2 field of the * level description is temporarily used to store the vertical sounding * significance, to simplify decoding. */ void sounding_pack_levels(); /** * Read data from a CSV input. * * Reading stops when one of Longitude, Latitude, Report or Date changes. * * @return true if some CSV data has been found, false on EOF */ bool from_csv(CSVReader& in); /// Output in CSV format void to_csv(CSVWriter& out) const; std::unique_ptr query_stations(const Query& query) const override; std::unique_ptr query_station_data(const Query& query) const override; std::unique_ptr query_data(const Query& query) const override; std::unique_ptr query_station_and_data(const Query& query) const; /// Output the CSV header static void csv_header(CSVWriter& out); /** * Get the message source type corresponding to the given report code */ static MessageType type_from_repmemo(const char* repmemo); /** * Get the report code corresponding to the given message source type */ static const char* repmemo_from_type(MessageType type); #include }; /** * Match adapter for impl::Message */ struct MatchedMsg : public Matched { const impl::Message& m; MatchedMsg(const impl::Message& r); ~MatchedMsg(); matcher::Result match_var_id(int val) const override; matcher::Result match_station_id(int val) const override; matcher::Result match_station_wmo(int block, int station=-1) const override; matcher::Result match_datetime(const DatetimeRange& range) const override; matcher::Result match_coords(const LatRange& latrange, const LonRange& lonrange) const override; matcher::Result match_rep_memo(const char* memo) const override; }; /** * Match adapter for Messages */ struct MatchedMessages : public Matched { const std::vector>& m; MatchedMessages(const std::vector>& m); ~MatchedMessages(); matcher::Result match_var_id(int val) const override; matcher::Result match_station_id(int val) const override; matcher::Result match_station_wmo(int block, int station=-1) const override; matcher::Result match_datetime(const DatetimeRange& range) const override; matcher::Result match_coords(const LatRange& latrange, const LonRange& lonrange) const override; matcher::Result match_rep_memo(const char* memo) const override; }; } } #endif dballe-8.6/dballe/msg/bulletin-test.cc0000644000175000017500000001713013554564112014656 00000000000000#include "tests.h" #include "bulletin.h" #include #include #include #include #include using namespace std; using namespace wreport; using namespace dballe; using namespace dballe::tests; namespace { struct MemStream { FILE* out = nullptr; char* buf = nullptr; size_t len = 0; MemStream() : out(open_memstream(&buf, &len)) { if (!out) throw std::system_error(errno, std::system_category(), "cannot open an in-memory stream"); } ~MemStream() { if (out) fclose(out); if (buf) free(buf); } void close() { fclose(out); out = nullptr; } operator FILE*() { return out; } }; class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("csv", []() { auto bulletin = BufrBulletin::create(); bulletin->edition_number = 4; bulletin->originating_centre = 0; bulletin->originating_subcentre = 0; bulletin->data_category = 0; bulletin->data_subcategory = 1; bulletin->data_subcategory_local = 2; bulletin->master_table_version_number = 14; bulletin->master_table_version_number_local = 0; bulletin->compression = false; bulletin->rep_year = 2015; bulletin->rep_month = 4; bulletin->rep_day = 25; bulletin->rep_hour = 12; bulletin->rep_minute = 30; bulletin->rep_second = 45; bulletin->load_tables(); // An integer bulletin->datadesc.push_back(WR_VAR(0, 1, 1)); bulletin->obtain_subset(0).store_variable_i(WR_VAR(0, 1, 1), 14); // A string bulletin->datadesc.push_back(WR_VAR(0, 1, 6)); bulletin->obtain_subset(0).store_variable_c(WR_VAR(0, 1, 6), "EZ1234"); // A decimal bulletin->datadesc.push_back(WR_VAR(0, 1, 14)); bulletin->obtain_subset(0).store_variable_d(WR_VAR(0, 1, 14), 3.14); // An undefined variable bulletin->datadesc.push_back(WR_VAR(0, 1, 2)); bulletin->obtain_subset(0).store_variable_undef(WR_VAR(0, 1, 2)); MemStream out; // Write the message out msg::BulletinCSVWriter writer(out); writer.output_bulletin(*bulletin); fflush(out); // Read it back and check it vector lines; { str::Split split(string(out.buf, out.len), "\n"); std::copy(split.begin(), split.end(), std::back_inserter(lines)); } wassert(actual(lines.size()) == 21u); wassert(actual(lines[ 0]) == R"("Field","Value")"); wassert(actual(lines[ 1]) == R"("master_table_number",0)"); wassert(actual(lines[ 2]) == R"("data_category",0)"); wassert(actual(lines[ 3]) == R"("data_subcategory",1)"); wassert(actual(lines[ 4]) == R"("data_subcategory_local",2)"); wassert(actual(lines[ 5]) == R"("originating_centre",0)"); wassert(actual(lines[ 6]) == R"("originating_subcentre",0)"); wassert(actual(lines[ 7]) == R"("update_sequence_number",0)"); wassert(actual(lines[ 8]) == R"("representative_time","2015-04-25 12:30:45")"); wassert(actual(lines[ 9]) == R"("encoding","bufr")"); wassert(actual(lines[10]) == R"("edition_number",4)"); wassert(actual(lines[11]) == R"("master_table_version_number",14)"); wassert(actual(lines[12]) == R"("master_table_version_number_local",0)"); wassert(actual(lines[13]) == R"("compression","false")"); wassert(actual(lines[14]) == R"("optional_section",)"); wassert(actual(lines[15]) == R"("subset",1)"); wassert(actual(lines[16]) == R"("B01001",14)"); wassert(actual(lines[17]) == R"("B01006","EZ1234")"); wassert(actual(lines[18]) == R"("B01014",3.14)"); wassert(actual(lines[19]) == R"("B01002",)"); // Empty line because the last line ends with a newline and // str::Split sees it as a trailing empty token wassert(actual(lines[20]) == R"()"); // Write the bulletin out again, to see that titles are not repeated writer.output_bulletin(*bulletin); fflush(out); lines.clear(); { str::Split split(string(out.buf, out.len), "\n"); std::copy(split.begin(), split.end(), std::back_inserter(lines)); } wassert(actual(lines.size()) == 40u); wassert(actual(lines[ 0]) == R"("Field","Value")"); wassert(actual(lines[ 1]) == R"("master_table_number",0)"); wassert(actual(lines[ 2]) == R"("data_category",0)"); wassert(actual(lines[ 3]) == R"("data_subcategory",1)"); wassert(actual(lines[ 4]) == R"("data_subcategory_local",2)"); wassert(actual(lines[ 5]) == R"("originating_centre",0)"); wassert(actual(lines[ 6]) == R"("originating_subcentre",0)"); wassert(actual(lines[ 7]) == R"("update_sequence_number",0)"); wassert(actual(lines[ 8]) == R"("representative_time","2015-04-25 12:30:45")"); wassert(actual(lines[ 9]) == R"("encoding","bufr")"); wassert(actual(lines[10]) == R"("edition_number",4)"); wassert(actual(lines[11]) == R"("master_table_version_number",14)"); wassert(actual(lines[12]) == R"("master_table_version_number_local",0)"); wassert(actual(lines[13]) == R"("compression","false")"); wassert(actual(lines[14]) == R"("optional_section",)"); wassert(actual(lines[15]) == R"("subset",1)"); wassert(actual(lines[16]) == R"("B01001",14)"); wassert(actual(lines[17]) == R"("B01006","EZ1234")"); wassert(actual(lines[18]) == R"("B01014",3.14)"); wassert(actual(lines[19]) == R"("B01002",)"); wassert(actual(lines[20]) == R"("master_table_number",0)"); wassert(actual(lines[21]) == R"("data_category",0)"); wassert(actual(lines[22]) == R"("data_subcategory",1)"); wassert(actual(lines[23]) == R"("data_subcategory_local",2)"); wassert(actual(lines[24]) == R"("originating_centre",0)"); wassert(actual(lines[25]) == R"("originating_subcentre",0)"); wassert(actual(lines[26]) == R"("update_sequence_number",0)"); wassert(actual(lines[27]) == R"("representative_time","2015-04-25 12:30:45")"); wassert(actual(lines[28]) == R"("encoding","bufr")"); wassert(actual(lines[29]) == R"("edition_number",4)"); wassert(actual(lines[30]) == R"("master_table_version_number",14)"); wassert(actual(lines[31]) == R"("master_table_version_number_local",0)"); wassert(actual(lines[32]) == R"("compression","false")"); wassert(actual(lines[33]) == R"("optional_section",)"); wassert(actual(lines[34]) == R"("subset",1)"); wassert(actual(lines[35]) == R"("B01001",14)"); wassert(actual(lines[36]) == R"("B01006","EZ1234")"); wassert(actual(lines[37]) == R"("B01014",3.14)"); wassert(actual(lines[38]) == R"("B01002",)"); // Empty line because the last line ends with a newline and // str::Split sees it as a trailing empty token wassert(actual(lines[39]) == R"()"); }); } } test("msg_bulletin"); } dballe-8.6/dballe/msg/wr_codec_generic-test.cc0000644000175000017500000002156313554564112016326 00000000000000#include "tests.h" #include "wr_codec.h" #include "msg.h" #include #include using namespace std; using namespace wreport; using namespace dballe; using namespace dballe::tests; namespace { class Tests : public TestCase { using TestCase::TestCase; void register_tests() override { add_method("empty", []() { // Try encoding and decoding an empty generic message unique_ptr importer = Importer::create(Encoding::BUFR); unique_ptr exporter = Exporter::create(Encoding::BUFR); impl::Messages msgs; msgs.emplace_back(make_shared()); // Export msg as a generic message BinaryMessage raw(Encoding::BUFR); raw.data = wcallchecked(exporter->to_binary(msgs)); // Parse it back impl::Messages msgs1 = wcallchecked(importer->from_binary(raw)); // Check that the data are the same notes::Collect c(cerr); int diffs = impl::msg::messages_diff(msgs, msgs1); if (diffs) dballe::tests::track_different_msgs(msgs, msgs1, "genericempty"); wassert(actual(diffs) == 0); }); add_method("known", []() { // Try encoding and decoding a generic message unique_ptr importer = Importer::create(Encoding::BUFR); unique_ptr exporter = Exporter::create(Encoding::BUFR); unique_ptr msg(new impl::Message); /* Fill up msg */ msg->set_press( 15, 45); msg->set_height_anem( 15, 45); msg->set_tot_snow( 15, 45); msg->set_visibility( 15, 45); msg->set_pres_wtr( 5, 45); msg->set_metar_wtr( 5, 45); msg->set_water_temp( 15, 45); msg->set_past_wtr1_3h( 2, 45); msg->set_past_wtr2_3h( 2, 45); msg->set_press_tend( 5, 45); msg->set_tot_prec24( 15, 45); msg->set_press_3h( 15, 45); msg->set_press_msl( 15, 45); msg->set_qnh( 15, 45); msg->set_temp_2m( 15, 45); msg->set_wet_temp_2m( 15, 45); msg->set_dewpoint_2m( 15, 45); msg->set_humidity( 15, 45); msg->set_wind_dir( 15, 45); msg->set_wind_speed( 15, 45); msg->set_ex_ccw_wind( 15, 45); msg->set_ex_cw_wind( 15, 45); msg->set_wind_gust_max_speed( 15, 45); msg->set_cloud_n( 3, 45); msg->set_cloud_nh( 10, 45); msg->set_cloud_hh( 3, 45); msg->set_cloud_cl( 3, 45); msg->set_cloud_cm( 3, 45); msg->set_cloud_ch( 3, 45); msg->set_cloud_n1( 3, 45); msg->set_cloud_c1( 3, 45); msg->set_cloud_h1( 3, 45); msg->set_cloud_n2( 3, 45); msg->set_cloud_c2( 3, 45); msg->set_cloud_h2( 3, 45); msg->set_cloud_n3( 3, 45); msg->set_cloud_c3( 3, 45); msg->set_cloud_h3( 3, 45); msg->set_cloud_n4( 3, 45); msg->set_cloud_c4( 3, 45); msg->set_cloud_h4( 3, 45); msg->set_block( 3, 45); msg->set_station( 3, 45); msg->set_flight_reg_no( "pippo", 45); msg->set_ident( "cippo", 45); msg->set_st_dir( 3, 45); msg->set_st_speed( 3, 45); msg->set_st_name( "ciop", 45); msg->set_st_name_icao( "cip", 45); msg->set_st_type( 1, 45); msg->set_wind_inst( 3, 45); msg->set_temp_precision( 1.23, 45); msg->set_sonde_type( 3, 45); msg->set_sonde_method( 3, 45); msg->set_navsys( 3, 45); msg->set_data_relay( 3, 45); msg->set_flight_roll( 3, 45); msg->set_latlon_spec( 3, 45); msg->set_datetime(Datetime(3, 3, 3, 3, 3, 0)); auto var = newvar(WR_VAR(0, 4, 1), 3); var->seta(newvar(WR_VAR(0, 33, 7), 45)); msg->station_data.set(std::move(var)); var = newvar(WR_VAR(0, 4, 2), 3); var->seta(newvar(WR_VAR(0, 33, 7), 45)); msg->station_data.set(std::move(var)); var = newvar(WR_VAR(0, 4, 3), 3); var->seta(newvar(WR_VAR(0, 33, 7), 45)); msg->station_data.set(std::move(var)); var = newvar(WR_VAR(0, 4, 4), 3); var->seta(newvar(WR_VAR(0, 33, 7), 45)); msg->station_data.set(std::move(var)); var = newvar(WR_VAR(0, 4, 5), 3); var->seta(newvar(WR_VAR(0, 33, 7), 45)); msg->station_data.set(std::move(var)); msg->set_latitude( 3, 45); msg->set_longitude( 3, 45); msg->set_height_station(3, 45); msg->set_height_baro( 3, 45); msg->set_flight_phase( 3, 45); msg->set_timesig( 3, 45); //CHECKED(dba_msg_set_flight_press( msg, 3, 45)); impl::Messages msgs; msgs.emplace_back(move(msg)); /* Export msg as a generic message */ BinaryMessage raw(Encoding::BUFR); raw.data = wcallchecked(exporter->to_binary(msgs)); //FILE* out = fopen("/tmp/zaza.bufr", "wb"); //fwrite(raw.data.data(), raw.data.size(), 1, out); //fclose(out); /* Parse it back */ impl::Messages msgs1 = wcallchecked(importer->from_binary(raw)); /* Check that the data are the same */ notes::Collect c(cerr); int diffs = impl::msg::messages_diff(msgs, msgs1); if (diffs) dballe::tests::track_different_msgs(msgs, msgs1, "generic2"); wassert(actual(diffs) == 0); }); add_method("attrs", []() { // Check that attributes are properly exported unique_ptr importer = Importer::create(Encoding::BUFR); unique_ptr exporter = Exporter::create(Encoding::BUFR); /* Create a new message */ unique_ptr msg(new impl::Message); msg->type = MessageType::GENERIC; // Set some metadata msg->set_datetime(Datetime(2006, 1, 19, 14, 50)); msg->set_latitude(50.0); msg->set_longitude(12.0); /* Create a variable to add to the message */ unique_ptr var = newvar(WR_VAR(0, 12, 101), 270.15); /* Add some attributes to the variable */ var->seta(newvar(WR_VAR(0, 33, 2), 1)); var->seta(newvar(WR_VAR(0, 33, 3), 2)); var->seta(newvar(WR_VAR(0, 33, 5), 3)); /* Add the variable to the message */ msg->set(Level(1), Trange::instant(), move(var)); /* Create a second variable to add to the message */ var = newvar(WR_VAR(0, 12, 102), 272.0); /* Add some attributes to the variable */ var->seta(newvar(WR_VAR(0, 33, 3), 1)); var->seta(newvar(WR_VAR(0, 33, 5), 2)); /* Add the variable to the message */ msg->set(Level(1), Trange::instant(), move(var)); impl::Messages msgs; msgs.emplace_back(move(msg)); // Encode the message BinaryMessage raw(Encoding::BUFR); raw.data = wcallchecked(exporter->to_binary(msgs)); // Decode the message impl::Messages msgs1 = wcallchecked(importer->from_binary(raw)); // Check that the data are the same notes::Collect c(cerr); int diffs = impl::msg::messages_diff(msgs, msgs1); if (diffs) dballe::tests::track_different_msgs(msgs, msgs1, "genericattr"); wassert(actual(diffs) == 0); }); add_method("doublememo", []() { // Test a bug in which B01194 ([SIM] Report mnemonic) appears twice // Import a synop message impl::Messages msgs = read_msgs("bufr/obs0-1.22.bufr", Encoding::BUFR); wassert(actual(msgs.size()) > 0); // Convert it to generic, with a 'ship' rep_memo impl::Message::downcast(msgs[0])->type = MessageType::GENERIC; impl::Message::downcast(msgs[0])->set_rep_memo("ship"); // Export it unique_ptr exporter = Exporter::create(Encoding::BUFR); unique_ptr bulletin = exporter->to_bulletin(msgs); // Ensure that B01194 only appears once wassert(actual(bulletin->subsets.size()) == 1u); unsigned count = 0; for (std::vector::const_iterator i = bulletin->subsets[0].begin(); i != bulletin->subsets[0].end(); ++i) { if (i->code() == WR_VAR(0, 1, 194)) ++count; } wassert(actual(count) == 1u); }); } } test("msg_wr_codec_generic"); } dballe-8.6/dballe/msg/msg-extravars.h0000644000175000017500000015423013554564124014536 00000000000000 /** Set the value of "Type of station" from a variable of type int */ inline void set_st_type(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 2, 1), val, conf); } /** Set the value of "Type of station" from a wreport::Var */ inline void set_st_type_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 1), val); } /** Get the "Type of station" physical value stored in the message */ inline const wreport::Var* get_st_type_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 1)); } /** Set the value of "Station or site name" from a variable of type const char* */ inline void set_st_name(const char* val, int conf=-1) { setc(Level(), Trange(), WR_VAR(0, 1, 19), val, conf); } /** Set the value of "Station or site name" from a wreport::Var */ inline void set_st_name_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 19), val); } /** Get the "Station or site name" physical value stored in the message */ inline const wreport::Var* get_st_name_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 19)); } /** Set the value of "ICAO location indicator" from a variable of type const char* */ inline void set_st_name_icao(const char* val, int conf=-1) { setc(Level(), Trange(), WR_VAR(0, 1, 63), val, conf); } /** Set the value of "ICAO location indicator" from a wreport::Var */ inline void set_st_name_icao_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 63), val); } /** Get the "ICAO location indicator" physical value stored in the message */ inline const wreport::Var* get_st_name_icao_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 63)); } /** Set the value of "Report mnemonic" from a variable of type const char* */ inline void set_rep_memo(const char* val, int conf=-1) { setc(Level(), Trange(), WR_VAR(0, 1, 194), val, conf); } /** Set the value of "Report mnemonic" from a wreport::Var */ inline void set_rep_memo_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 194), val); } /** Get the "Report mnemonic" physical value stored in the message */ inline const wreport::Var* get_rep_memo_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 194)); } /** Set the value of "Air quality observing station local code" from a variable of type int */ inline void set_poll_lcode(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 212), val, conf); } /** Set the value of "Air quality observing station local code" from a wreport::Var */ inline void set_poll_lcode_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 212), val); } /** Get the "Air quality observing station local code" physical value stored in the message */ inline const wreport::Var* get_poll_lcode_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 212)); } /** Set the value of "Airbase air quality observing station code" from a variable of type int */ inline void set_poll_scode(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 213), val, conf); } /** Set the value of "Airbase air quality observing station code" from a wreport::Var */ inline void set_poll_scode_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 213), val); } /** Get the "Airbase air quality observing station code" physical value stored in the message */ inline const wreport::Var* get_poll_scode_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 213)); } /** Set the value of "GEMS air quality observing station code" from a variable of type int */ inline void set_poll_gemscode(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 214), val, conf); } /** Set the value of "GEMS air quality observing station code" from a wreport::Var */ inline void set_poll_gemscode_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 214), val); } /** Get the "GEMS air quality observing station code" physical value stored in the message */ inline const wreport::Var* get_poll_gemscode_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 214)); } /** Set the value of "Air quality observing station dominant emission source" from a variable of type int */ inline void set_poll_source(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 215), val, conf); } /** Set the value of "Air quality observing station dominant emission source" from a wreport::Var */ inline void set_poll_source_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 215), val); } /** Get the "Air quality observing station dominant emission source" physical value stored in the message */ inline const wreport::Var* get_poll_source_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 215)); } /** Set the value of "Air quality observing station area type" from a variable of type int */ inline void set_poll_atype(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 216), val, conf); } /** Set the value of "Air quality observing station area type" from a wreport::Var */ inline void set_poll_atype_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 216), val); } /** Get the "Air quality observing station area type" physical value stored in the message */ inline const wreport::Var* get_poll_atype_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 216)); } /** Set the value of "Air quality observing station terrain type" from a variable of type int */ inline void set_poll_ttype(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 217), val, conf); } /** Set the value of "Air quality observing station terrain type" from a wreport::Var */ inline void set_poll_ttype_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 217), val); } /** Get the "Air quality observing station terrain type" physical value stored in the message */ inline const wreport::Var* get_poll_ttype_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 217)); } /** Set the value of "Aircraft registration number or other identification" from a variable of type const char* */ inline void set_flight_reg_no(const char* val, int conf=-1) { setc(Level(), Trange(), WR_VAR(0, 1, 8), val, conf); } /** Set the value of "Aircraft registration number or other identification" from a wreport::Var */ inline void set_flight_reg_no_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 8), val); } /** Get the "Aircraft registration number or other identification" physical value stored in the message */ inline const wreport::Var* get_flight_reg_no_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 8)); } /** Set the value of "Phase of aircraft flight" from a variable of type int */ inline void set_flight_phase(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 8, 4), val, conf); } /** Set the value of "Phase of aircraft flight" from a wreport::Var */ inline void set_flight_phase_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 8, 4), val); } /** Get the "Phase of aircraft flight" physical value stored in the message */ inline const wreport::Var* get_flight_phase_var() const { return get(Level(), Trange(), WR_VAR(0, 8, 4)); } /** Set the value of "Aircraft roll angle" from a variable of type double */ inline void set_flight_roll(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 2, 63), val, conf); } /** Set the value of "Aircraft roll angle" from a wreport::Var */ inline void set_flight_roll_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 63), val); } /** Get the "Aircraft roll angle" physical value stored in the message */ inline const wreport::Var* get_flight_roll_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 63)); } /** Set the value of "Aircraft navigational system" from a variable of type int */ inline void set_navsys(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 2, 61), val, conf); } /** Set the value of "Aircraft navigational system" from a wreport::Var */ inline void set_navsys_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 61), val); } /** Get the "Aircraft navigational system" physical value stored in the message */ inline const wreport::Var* get_navsys_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 61)); } /** Set the value of "Aircraft data relay system" from a variable of type int */ inline void set_data_relay(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 2, 62), val, conf); } /** Set the value of "Aircraft data relay system" from a wreport::Var */ inline void set_data_relay_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 62), val); } /** Get the "Aircraft data relay system" physical value stored in the message */ inline const wreport::Var* get_data_relay_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 62)); } /** Set the value of "Type of instrumentation for wind measurement" from a variable of type int */ inline void set_wind_inst(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 2, 2), val, conf); } /** Set the value of "Type of instrumentation for wind measurement" from a wreport::Var */ inline void set_wind_inst_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 2), val); } /** Get the "Type of instrumentation for wind measurement" physical value stored in the message */ inline const wreport::Var* get_wind_inst_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 2)); } /** Set the value of "Precision of temperature observation" from a variable of type double */ inline void set_temp_precision(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 2, 5), val, conf); } /** Set the value of "Precision of temperature observation" from a wreport::Var */ inline void set_temp_precision_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 5), val); } /** Get the "Precision of temperature observation" physical value stored in the message */ inline const wreport::Var* get_temp_precision_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 5)); } /** Set the value of "Original specification of latitude/longitude" from a variable of type int */ inline void set_latlon_spec(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 2, 70), val, conf); } /** Set the value of "Original specification of latitude/longitude" from a wreport::Var */ inline void set_latlon_spec_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 2, 70), val); } /** Get the "Original specification of latitude/longitude" physical value stored in the message */ inline const wreport::Var* get_latlon_spec_var() const { return get(Level(), Trange(), WR_VAR(0, 2, 70)); } /** Set the value of "Time significance" from a variable of type int */ inline void set_timesig(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 8, 21), val, conf); } /** Set the value of "Time significance" from a wreport::Var */ inline void set_timesig_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 8, 21), val); } /** Get the "Time significance" physical value stored in the message */ inline const wreport::Var* get_timesig_var() const { return get(Level(), Trange(), WR_VAR(0, 8, 21)); } /** Set the value of "WMO block number" from a variable of type int */ inline void set_block(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 1), val, conf); } /** Set the value of "WMO block number" from a wreport::Var */ inline void set_block_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 1), val); } /** Get the "WMO block number" physical value stored in the message */ inline const wreport::Var* get_block_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 1)); } /** Set the value of "WMO station number" from a variable of type int */ inline void set_station(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 1, 2), val, conf); } /** Set the value of "WMO station number" from a wreport::Var */ inline void set_station_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 2), val); } /** Get the "WMO station number" physical value stored in the message */ inline const wreport::Var* get_station_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 2)); } /** Set the value of "Mobile station identifier" from a variable of type const char* */ inline void set_ident(const char* val, int conf=-1) { setc(Level(), Trange(), WR_VAR(0, 1, 11), val, conf); } /** Set the value of "Mobile station identifier" from a wreport::Var */ inline void set_ident_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 1, 11), val); } /** Get the "Mobile station identifier" physical value stored in the message */ inline const wreport::Var* get_ident_var() const { return get(Level(), Trange(), WR_VAR(0, 1, 11)); } /** Set the value of "Year of the observation" from a variable of type int */ inline void set_year(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 1), val, conf); } /** Set the value of "Year of the observation" from a wreport::Var */ inline void set_year_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 1), val); } /** Get the "Year of the observation" physical value stored in the message */ inline const wreport::Var* get_year_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 1)); } /** Set the value of "Month of the observation" from a variable of type int */ inline void set_month(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 2), val, conf); } /** Set the value of "Month of the observation" from a wreport::Var */ inline void set_month_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 2), val); } /** Get the "Month of the observation" physical value stored in the message */ inline const wreport::Var* get_month_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 2)); } /** Set the value of "Day of the observation" from a variable of type int */ inline void set_day(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 3), val, conf); } /** Set the value of "Day of the observation" from a wreport::Var */ inline void set_day_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 3), val); } /** Get the "Day of the observation" physical value stored in the message */ inline const wreport::Var* get_day_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 3)); } /** Set the value of "Hour of the observation" from a variable of type int */ inline void set_hour(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 4), val, conf); } /** Set the value of "Hour of the observation" from a wreport::Var */ inline void set_hour_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 4), val); } /** Get the "Hour of the observation" physical value stored in the message */ inline const wreport::Var* get_hour_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 4)); } /** Set the value of "Minute of the observation" from a variable of type int */ inline void set_minute(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 5), val, conf); } /** Set the value of "Minute of the observation" from a wreport::Var */ inline void set_minute_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 5), val); } /** Get the "Minute of the observation" physical value stored in the message */ inline const wreport::Var* get_minute_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 5)); } /** Set the value of "Second of the observation" from a variable of type int */ inline void set_second(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 4, 6), val, conf); } /** Set the value of "Second of the observation" from a wreport::Var */ inline void set_second_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 4, 6), val); } /** Get the "Second of the observation" physical value stored in the message */ inline const wreport::Var* get_second_var() const { return get(Level(), Trange(), WR_VAR(0, 4, 6)); } /** Set the value of "Latitude of the station" from a variable of type double */ inline void set_latitude(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 5, 1), val, conf); } /** Set the value of "Latitude of the station" from a wreport::Var */ inline void set_latitude_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 5, 1), val); } /** Get the "Latitude of the station" physical value stored in the message */ inline const wreport::Var* get_latitude_var() const { return get(Level(), Trange(), WR_VAR(0, 5, 1)); } /** Set the value of "Longiture of the station" from a variable of type double */ inline void set_longitude(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 6, 1), val, conf); } /** Set the value of "Longiture of the station" from a wreport::Var */ inline void set_longitude_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 6, 1), val); } /** Get the "Longiture of the station" physical value stored in the message */ inline const wreport::Var* get_longitude_var() const { return get(Level(), Trange(), WR_VAR(0, 6, 1)); } /** Set the value of "Height of station" from a variable of type double */ inline void set_height_station(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 7, 30), val, conf); } /** Set the value of "Height of station" from a wreport::Var */ inline void set_height_station_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 7, 30), val); } /** Get the "Height of station" physical value stored in the message */ inline const wreport::Var* get_height_station_var() const { return get(Level(), Trange(), WR_VAR(0, 7, 30)); } /** Set the value of "Height of barometer above mean sea level" from a variable of type double */ inline void set_height_baro(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 7, 31), val, conf); } /** Set the value of "Height of barometer above mean sea level" from a wreport::Var */ inline void set_height_baro_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 7, 31), val); } /** Get the "Height of barometer above mean sea level" physical value stored in the message */ inline const wreport::Var* get_height_baro_var() const { return get(Level(), Trange(), WR_VAR(0, 7, 31)); } /** Set the value of "Height of release of sonde above msl" from a variable of type double */ inline void set_height_release(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 7, 7), val, conf); } /** Set the value of "Height of release of sonde above msl" from a wreport::Var */ inline void set_height_release_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 7, 7), val); } /** Get the "Height of release of sonde above msl" physical value stored in the message */ inline const wreport::Var* get_height_release_var() const { return get(Level(), Trange(), WR_VAR(0, 7, 7)); } /** Set the value of "Station elevation quality mark (for mobile stations)" from a variable of type int */ inline void set_station_height_quality(int val, int conf=-1) { seti(Level(), Trange(), WR_VAR(0, 33, 24), val, conf); } /** Set the value of "Station elevation quality mark (for mobile stations)" from a wreport::Var */ inline void set_station_height_quality_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 33, 24), val); } /** Get the "Station elevation quality mark (for mobile stations)" physical value stored in the message */ inline const wreport::Var* get_station_height_quality_var() const { return get(Level(), Trange(), WR_VAR(0, 33, 24)); } /** Set the value of "Isobaric surface" from a variable of type double */ inline void set_isobaric_surface(double val, int conf=-1) { setd(Level(), Trange(), WR_VAR(0, 7, 4), val, conf); } /** Set the value of "Isobaric surface" from a wreport::Var */ inline void set_isobaric_surface_var(const wreport::Var& val) { set(Level(), Trange(), WR_VAR(0, 7, 4), val); } /** Get the "Isobaric surface" physical value stored in the message */ inline const wreport::Var* get_isobaric_surface_var() const { return get(Level(), Trange(), WR_VAR(0, 7, 4)); } /** Set the value of "Direction of motion of moving observing platform" from a variable of type int */ inline void set_st_dir(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 1, 12), val, conf); } /** Set the value of "Direction of motion of moving observing platform" from a wreport::Var */ inline void set_st_dir_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 1, 12), val); } /** Get the "Direction of motion of moving observing platform" physical value stored in the message */ inline const wreport::Var* get_st_dir_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 1, 12)); } /** Set the value of "Speed of motion of moving observing platform" from a variable of type int */ inline void set_st_speed(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 1, 13), val, conf); } /** Set the value of "Speed of motion of moving observing platform" from a wreport::Var */ inline void set_st_speed_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 1, 13), val); } /** Get the "Speed of motion of moving observing platform" physical value stored in the message */ inline const wreport::Var* get_st_speed_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 1, 13)); } /** Set the value of "Type of measuring equipment used" from a variable of type int */ inline void set_meas_equip_type(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 2, 3), val, conf); } /** Set the value of "Type of measuring equipment used" from a wreport::Var */ inline void set_meas_equip_type_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 2, 3), val); } /** Get the "Type of measuring equipment used" physical value stored in the message */ inline const wreport::Var* get_meas_equip_type_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 2, 3)); } /** Set the value of "Radiosonde type" from a variable of type int */ inline void set_sonde_type(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 2, 11), val, conf); } /** Set the value of "Radiosonde type" from a wreport::Var */ inline void set_sonde_type_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 2, 11), val); } /** Get the "Radiosonde type" physical value stored in the message */ inline const wreport::Var* get_sonde_type_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 2, 11)); } /** Set the value of "Radiosonde computational method" from a variable of type int */ inline void set_sonde_method(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 2, 12), val, conf); } /** Set the value of "Radiosonde computational method" from a wreport::Var */ inline void set_sonde_method_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 2, 12), val); } /** Get the "Radiosonde computational method" physical value stored in the message */ inline const wreport::Var* get_sonde_method_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 2, 12)); } /** Set the value of "Solar and infrared radiation correction" from a variable of type int */ inline void set_sonde_correction(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 2, 13), val, conf); } /** Set the value of "Solar and infrared radiation correction" from a wreport::Var */ inline void set_sonde_correction_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 2, 13), val); } /** Get the "Solar and infrared radiation correction" physical value stored in the message */ inline const wreport::Var* get_sonde_correction_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 2, 13)); } /** Set the value of "Tracking technique/status of system used" from a variable of type int */ inline void set_sonde_tracking(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 2, 14), val, conf); } /** Set the value of "Tracking technique/status of system used" from a wreport::Var */ inline void set_sonde_tracking_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 2, 14), val); } /** Get the "Tracking technique/status of system used" physical value stored in the message */ inline const wreport::Var* get_sonde_tracking_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 2, 14)); } /** Set the value of "Pressure at ground level" from a variable of type double */ inline void set_press(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 10, 4), val, conf); } /** Set the value of "Pressure at ground level" from a wreport::Var */ inline void set_press_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 10, 4), val); } /** Get the "Pressure at ground level" physical value stored in the message */ inline const wreport::Var* get_press_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 10, 4)); } /** Set the value of "3 hour pressure change at ground level" from a variable of type double */ inline void set_press_3h(double val, int conf=-1) { setd(Level(1), Trange(4,0,10800), WR_VAR(0, 10, 60), val, conf); } /** Set the value of "3 hour pressure change at ground level" from a wreport::Var */ inline void set_press_3h_var(const wreport::Var& val) { set(Level(1), Trange(4,0,10800), WR_VAR(0, 10, 60), val); } /** Get the "3 hour pressure change at ground level" physical value stored in the message */ inline const wreport::Var* get_press_3h_var() const { return get(Level(1), Trange(4,0,10800), WR_VAR(0, 10, 60)); } /** Set the value of "24 hour pressure change at ground level" from a variable of type double */ inline void set_press_24h(double val, int conf=-1) { setd(Level(1), Trange(4,0,86400), WR_VAR(0, 10, 60), val, conf); } /** Set the value of "24 hour pressure change at ground level" from a wreport::Var */ inline void set_press_24h_var(const wreport::Var& val) { set(Level(1), Trange(4,0,86400), WR_VAR(0, 10, 60), val); } /** Get the "24 hour pressure change at ground level" physical value stored in the message */ inline const wreport::Var* get_press_24h_var() const { return get(Level(1), Trange(4,0,86400), WR_VAR(0, 10, 60)); } /** Set the value of "Sea/water surface temperature" from a variable of type double */ inline void set_water_temp(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 22, 43), val, conf); } /** Set the value of "Sea/water surface temperature" from a wreport::Var */ inline void set_water_temp_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 22, 43), val); } /** Get the "Sea/water surface temperature" physical value stored in the message */ inline const wreport::Var* get_water_temp_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 22, 43)); } /** Set the value of "Anemometer height" from a variable of type double */ inline void set_height_anem(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 10, 197), val, conf); } /** Set the value of "Anemometer height" from a wreport::Var */ inline void set_height_anem_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 10, 197), val); } /** Get the "Anemometer height" physical value stored in the message */ inline const wreport::Var* get_height_anem_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 10, 197)); } /** Set the value of "Characteristic of pressure tendency" from a variable of type double */ inline void set_press_tend(double val, int conf=-1) { setd(Level(1), Trange(205,0,10800), WR_VAR(0, 10, 63), val, conf); } /** Set the value of "Characteristic of pressure tendency" from a wreport::Var */ inline void set_press_tend_var(const wreport::Var& val) { set(Level(1), Trange(205,0,10800), WR_VAR(0, 10, 63), val); } /** Get the "Characteristic of pressure tendency" physical value stored in the message */ inline const wreport::Var* get_press_tend_var() const { return get(Level(1), Trange(205,0,10800), WR_VAR(0, 10, 63)); } /** Set the value of "Visibility" from a variable of type double */ inline void set_visibility(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 20, 1), val, conf); } /** Set the value of "Visibility" from a wreport::Var */ inline void set_visibility_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 20, 1), val); } /** Get the "Visibility" physical value stored in the message */ inline const wreport::Var* get_visibility_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 20, 1)); } /** Set the value of "Present weather" from a variable of type int */ inline void set_pres_wtr(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 20, 3), val, conf); } /** Set the value of "Present weather" from a wreport::Var */ inline void set_pres_wtr_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 20, 3), val); } /** Get the "Present weather" physical value stored in the message */ inline const wreport::Var* get_pres_wtr_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 20, 3)); } /** Set the value of "Past weather (1 - 3h)" from a variable of type int */ inline void set_past_wtr1_3h(int val, int conf=-1) { seti(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 4), val, conf); } /** Set the value of "Past weather (1 - 3h)" from a wreport::Var */ inline void set_past_wtr1_3h_var(const wreport::Var& val) { set(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 4), val); } /** Get the "Past weather (1 - 3h)" physical value stored in the message */ inline const wreport::Var* get_past_wtr1_3h_var() const { return get(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 4)); } /** Set the value of "Past weather (1 - 6h)" from a variable of type int */ inline void set_past_wtr1_6h(int val, int conf=-1) { seti(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 4), val, conf); } /** Set the value of "Past weather (1 - 6h)" from a wreport::Var */ inline void set_past_wtr1_6h_var(const wreport::Var& val) { set(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 4), val); } /** Get the "Past weather (1 - 6h)" physical value stored in the message */ inline const wreport::Var* get_past_wtr1_6h_var() const { return get(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 4)); } /** Set the value of "Past weather (2 - 3h)" from a variable of type int */ inline void set_past_wtr2_3h(int val, int conf=-1) { seti(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 5), val, conf); } /** Set the value of "Past weather (2 - 3h)" from a wreport::Var */ inline void set_past_wtr2_3h_var(const wreport::Var& val) { set(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 5), val); } /** Get the "Past weather (2 - 3h)" physical value stored in the message */ inline const wreport::Var* get_past_wtr2_3h_var() const { return get(Level(1), Trange(205,0,10800), WR_VAR(0, 20, 5)); } /** Set the value of "Past weather (2 - 6h)" from a variable of type int */ inline void set_past_wtr2_6h(int val, int conf=-1) { seti(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 5), val, conf); } /** Set the value of "Past weather (2 - 6h)" from a wreport::Var */ inline void set_past_wtr2_6h_var(const wreport::Var& val) { set(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 5), val); } /** Get the "Past weather (2 - 6h)" physical value stored in the message */ inline const wreport::Var* get_past_wtr2_6h_var() const { return get(Level(1), Trange(205,0,21600), WR_VAR(0, 20, 5)); } /** Set the value of "General weather indicator (TAF/METAR)" from a variable of type int */ inline void set_metar_wtr(int val, int conf=-1) { seti(Level(1), Trange(254,0,0), WR_VAR(0, 20, 9), val, conf); } /** Set the value of "General weather indicator (TAF/METAR)" from a wreport::Var */ inline void set_metar_wtr_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 20, 9), val); } /** Get the "General weather indicator (TAF/METAR)" physical value stored in the message */ inline const wreport::Var* get_metar_wtr_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 20, 9)); } /** Set the value of "Total precipitation in the last hour" from a variable of type double */ inline void set_tot_prec1(double val, int conf=-1) { setd(Level(1), Trange(1,0,3600), WR_VAR(0, 13, 11), val, conf); } /** Set the value of "Total precipitation in the last hour" from a wreport::Var */ inline void set_tot_prec1_var(const wreport::Var& val) { set(Level(1), Trange(1,0,3600), WR_VAR(0, 13, 11), val); } /** Get the "Total precipitation in the last hour" physical value stored in the message */ inline const wreport::Var* get_tot_prec1_var() const { return get(Level(1), Trange(1,0,3600), WR_VAR(0, 13, 11)); } /** Set the value of "Total precipitation in the last 3 hours" from a variable of type double */ inline void set_tot_prec3(double val, int conf=-1) { setd(Level(1), Trange(1,0,10800), WR_VAR(0, 13, 11), val, conf); } /** Set the value of "Total precipitation in the last 3 hours" from a wreport::Var */ inline void set_tot_prec3_var(const wreport::Var& val) { set(Level(1), Trange(1,0,10800), WR_VAR(0, 13, 11), val); } /** Get the "Total precipitation in the last 3 hours" physical value stored in the message */ inline const wreport::Var* get_tot_prec3_var() const { return get(Level(1), Trange(1,0,10800), WR_VAR(0, 13, 11)); } /** Set the value of "Total precipitation in the last 6 hours" from a variable of type double */ inline void set_tot_prec6(double val, int conf=-1) { setd(Level(1), Trange(1,0,21600), WR_VAR(0, 13, 11), val, conf); } /** Set the value of "Total precipitation in the last 6 hours" from a wreport::Var */ inline void set_tot_prec6_var(const wreport::Var& val) { set(Level(1), Trange(1,0,21600), WR_VAR(0, 13, 11), val); } /** Get the "Total precipitation in the last 6 hours" physical value stored in the message */ inline const wreport::Var* get_tot_prec6_var() const { return get(Level(1), Trange(1,0,21600), WR_VAR(0, 13, 11)); } /** Set the value of "Total precipitation in the last 12 hours" from a variable of type double */ inline void set_tot_prec12(double val, int conf=-1) { setd(Level(1), Trange(1,0,43200), WR_VAR(0, 13, 11), val, conf); } /** Set the value of "Total precipitation in the last 12 hours" from a wreport::Var */ inline void set_tot_prec12_var(const wreport::Var& val) { set(Level(1), Trange(1,0,43200), WR_VAR(0, 13, 11), val); } /** Get the "Total precipitation in the last 12 hours" physical value stored in the message */ inline const wreport::Var* get_tot_prec12_var() const { return get(Level(1), Trange(1,0,43200), WR_VAR(0, 13, 11)); } /** Set the value of "Total precipitation in the last 24 hours" from a variable of type double */ inline void set_tot_prec24(double val, int conf=-1) { setd(Level(1), Trange(1,0,86400), WR_VAR(0, 13, 11), val, conf); } /** Set the value of "Total precipitation in the last 24 hours" from a wreport::Var */ inline void set_tot_prec24_var(const wreport::Var& val) { set(Level(1), Trange(1,0,86400), WR_VAR(0, 13, 11), val); } /** Get the "Total precipitation in the last 24 hours" physical value stored in the message */ inline const wreport::Var* get_tot_prec24_var() const { return get(Level(1), Trange(1,0,86400), WR_VAR(0, 13, 11)); } /** Set the value of "Total show depth" from a variable of type double */ inline void set_tot_snow(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 13, 13), val, conf); } /** Set the value of "Total show depth" from a wreport::Var */ inline void set_tot_snow_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 13, 13), val); } /** Get the "Total show depth" physical value stored in the message */ inline const wreport::Var* get_tot_snow_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 13, 13)); } /** Set the value of "State of ground (with or without snow)" from a variable of type double */ inline void set_state_ground(double val, int conf=-1) { setd(Level(1), Trange(254,0,0), WR_VAR(0, 20, 62), val, conf); } /** Set the value of "State of ground (with or without snow)" from a wreport::Var */ inline void set_state_ground_var(const wreport::Var& val) { set(Level(1), Trange(254,0,0), WR_VAR(0, 20, 62), val); } /** Get the "State of ground (with or without snow)" physical value stored in the message */ inline const wreport::Var* get_state_ground_var() const { return get(Level(1), Trange(254,0,0), WR_VAR(0, 20, 62)); } /** Set the value of "Pressure reduced to mean sea level" from a variable of type double */ inline void set_press_msl(double val, int conf=-1) { setd(Level(101), Trange(254,0,0), WR_VAR(0, 10, 51), val, conf); } /** Set the value of "Pressure reduced to mean sea level" from a wreport::Var */ inline void set_press_msl_var(const wreport::Var& val) { set(Level(101), Trange(254,0,0), WR_VAR(0, 10, 51), val); } /** Get the "Pressure reduced to mean sea level" physical value stored in the message */ inline const wreport::Var* get_press_msl_var() const { return get(Level(101), Trange(254,0,0), WR_VAR(0, 10, 51)); } /** Set the value of "Altimeter setting (QNH)" from a variable of type double */ inline void set_qnh(double val, int conf=-1) { setd(Level(103,2000), Trange(254,0,0), WR_VAR(0, 10, 52), val, conf); } /** Set the value of "Altimeter setting (QNH)" from a wreport::Var */ inline void set_qnh_var(const wreport::Var& val) { set(Level(103,2000), Trange(254,0,0), WR_VAR(0, 10, 52), val); } /** Get the "Altimeter setting (QNH)" physical value stored in the message */ inline const wreport::Var* get_qnh_var() const { return get(Level(103,2000), Trange(254,0,0), WR_VAR(0, 10, 52)); } /** Set the value of "Temperature at 2 metres above ground" from a variable of type double */ inline void set_temp_2m(double val, int conf=-1) { setd(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 101), val, conf); } /** Set the value of "Temperature at 2 metres above ground" from a wreport::Var */ inline void set_temp_2m_var(const wreport::Var& val) { set(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 101), val); } /** Get the "Temperature at 2 metres above ground" physical value stored in the message */ inline const wreport::Var* get_temp_2m_var() const { return get(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 101)); } /** Set the value of "Wet bulb temperature at 2 metres above ground" from a variable of type double */ inline void set_wet_temp_2m(double val, int conf=-1) { setd(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 102), val, conf); } /** Set the value of "Wet bulb temperature at 2 metres above ground" from a wreport::Var */ inline void set_wet_temp_2m_var(const wreport::Var& val) { set(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 102), val); } /** Get the "Wet bulb temperature at 2 metres above ground" physical value stored in the message */ inline const wreport::Var* get_wet_temp_2m_var() const { return get(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 102)); } /** Set the value of "Dew point at 2 metres above ground" from a variable of type double */ inline void set_dewpoint_2m(double val, int conf=-1) { setd(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 103), val, conf); } /** Set the value of "Dew point at 2 metres above ground" from a wreport::Var */ inline void set_dewpoint_2m_var(const wreport::Var& val) { set(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 103), val); } /** Get the "Dew point at 2 metres above ground" physical value stored in the message */ inline const wreport::Var* get_dewpoint_2m_var() const { return get(Level(103,2000), Trange(254,0,0), WR_VAR(0, 12, 103)); } /** Set the value of "Humidity at 2 metres above ground" from a variable of type double */ inline void set_humidity(double val, int conf=-1) { setd(Level(103,2000), Trange(254,0,0), WR_VAR(0, 13, 3), val, conf); } /** Set the value of "Humidity at 2 metres above ground" from a wreport::Var */ inline void set_humidity_var(const wreport::Var& val) { set(Level(103,2000), Trange(254,0,0), WR_VAR(0, 13, 3), val); } /** Get the "Humidity at 2 metres above ground" physical value stored in the message */ inline const wreport::Var* get_humidity_var() const { return get(Level(103,2000), Trange(254,0,0), WR_VAR(0, 13, 3)); } /** Set the value of "Wind direction at 10 metres above ground" from a variable of type double */ inline void set_wind_dir(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 1), val, conf); } /** Set the value of "Wind direction at 10 metres above ground" from a wreport::Var */ inline void set_wind_dir_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 1), val); } /** Get the "Wind direction at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_wind_dir_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 1)); } /** Set the value of "Wind speed at 10 metres above ground" from a variable of type double */ inline void set_wind_speed(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 2), val, conf); } /** Set the value of "Wind speed at 10 metres above ground" from a wreport::Var */ inline void set_wind_speed_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 2), val); } /** Get the "Wind speed at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_wind_speed_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 2)); } /** Set the value of "Maximum wind gust speed at 10 metres above ground" from a variable of type double */ inline void set_wind_gust_max_speed(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 41), val, conf); } /** Set the value of "Maximum wind gust speed at 10 metres above ground" from a wreport::Var */ inline void set_wind_gust_max_speed_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 41), val); } /** Get the "Maximum wind gust speed at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_wind_gust_max_speed_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 41)); } /** Set the value of "Maximum wind gust direction at 10 metres above ground" from a variable of type double */ inline void set_wind_gust_max_dir(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 43), val, conf); } /** Set the value of "Maximum wind gust direction at 10 metres above ground" from a wreport::Var */ inline void set_wind_gust_max_dir_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 43), val); } /** Get the "Maximum wind gust direction at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_wind_gust_max_dir_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 43)); } /** Set the value of "Extreme counterclockwise wind direction of a variable wind at 10 metres above ground" from a variable of type double */ inline void set_ex_ccw_wind(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 16), val, conf); } /** Set the value of "Extreme counterclockwise wind direction of a variable wind at 10 metres above ground" from a wreport::Var */ inline void set_ex_ccw_wind_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 16), val); } /** Get the "Extreme counterclockwise wind direction of a variable wind at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_ex_ccw_wind_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 16)); } /** Set the value of "Extreme clockwise wind direction of a variable wind at 10 metres above ground" from a variable of type double */ inline void set_ex_cw_wind(double val, int conf=-1) { setd(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 17), val, conf); } /** Set the value of "Extreme clockwise wind direction of a variable wind at 10 metres above ground" from a wreport::Var */ inline void set_ex_cw_wind_var(const wreport::Var& val) { set(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 17), val); } /** Get the "Extreme clockwise wind direction of a variable wind at 10 metres above ground" physical value stored in the message */ inline const wreport::Var* get_ex_cw_wind_var() const { return get(Level(103,10000), Trange(254,0,0), WR_VAR(0, 11, 17)); } /** Set the value of "Total cloud cover (N)" from a variable of type int */ inline void set_cloud_n(int val, int conf=-1) { seti(Level(256), Trange(254,0,0), WR_VAR(0, 20, 10), val, conf); } /** Set the value of "Total cloud cover (N)" from a wreport::Var */ inline void set_cloud_n_var(const wreport::Var& val) { set(Level(256), Trange(254,0,0), WR_VAR(0, 20, 10), val); } /** Get the "Total cloud cover (N)" physical value stored in the message */ inline const wreport::Var* get_cloud_n_var() const { return get(Level(256), Trange(254,0,0), WR_VAR(0, 20, 10)); } /** Set the value of "Cloud amount (NH)" from a variable of type int */ inline void set_cloud_nh(int val, int conf=-1) { seti(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 11), val, conf); } /** Set the value of "Cloud amount (NH)" from a wreport::Var */ inline void set_cloud_nh_var(const wreport::Var& val) { set(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 11), val); } /** Get the "Cloud amount (NH)" physical value stored in the message */ inline const wreport::Var* get_cloud_nh_var() const { return get(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 11)); } /** Set the value of "Height of base of cloud (HH)" from a variable of type double */ inline void set_cloud_hh(double val, int conf=-1) { setd(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 13), val, conf); } /** Set the value of "Height of base of cloud (HH)" from a wreport::Var */ inline void set_cloud_hh_var(const wreport::Var& val) { set(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 13), val); } /** Get the "Height of base of cloud (HH)" physical value stored in the message */ inline const wreport::Var* get_cloud_hh_var() const { return get(Level(256,MISSING_INT,258,0), Trange(254,0,0), WR_VAR(0, 20, 13)); } /** Set the value of "Cloud type (CL)" from a variable of type int */ inline void set_cloud_cl(int val, int conf=-1) { seti(Level(256,MISSING_INT,258,1), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud type (CL)" from a wreport::Var */ inline void set_cloud_cl_var(const wreport::Var& val) { set(Level(256,MISSING_INT,258,1), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud type (CL)" physical value stored in the message */ inline const wreport::Var* get_cloud_cl_var() const { return get(Level(256,MISSING_INT,258,1), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Cloud type (CM)" from a variable of type int */ inline void set_cloud_cm(int val, int conf=-1) { seti(Level(256,MISSING_INT,258,2), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud type (CM)" from a wreport::Var */ inline void set_cloud_cm_var(const wreport::Var& val) { set(Level(256,MISSING_INT,258,2), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud type (CM)" physical value stored in the message */ inline const wreport::Var* get_cloud_cm_var() const { return get(Level(256,MISSING_INT,258,2), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Cloud type (CH)" from a variable of type int */ inline void set_cloud_ch(int val, int conf=-1) { seti(Level(256,MISSING_INT,258,3), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud type (CH)" from a wreport::Var */ inline void set_cloud_ch_var(const wreport::Var& val) { set(Level(256,MISSING_INT,258,3), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud type (CH)" physical value stored in the message */ inline const wreport::Var* get_cloud_ch_var() const { return get(Level(256,MISSING_INT,258,3), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Cloud amount (N1)" from a variable of type int */ inline void set_cloud_n1(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 11), val, conf); } /** Set the value of "Cloud amount (N1)" from a wreport::Var */ inline void set_cloud_n1_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 11), val); } /** Get the "Cloud amount (N1)" physical value stored in the message */ inline const wreport::Var* get_cloud_n1_var() const { return get(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 11)); } /** Set the value of "Cloud amount (C1)" from a variable of type int */ inline void set_cloud_c1(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud amount (C1)" from a wreport::Var */ inline void set_cloud_c1_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud amount (C1)" physical value stored in the message */ inline const wreport::Var* get_cloud_c1_var() const { return get(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Height of base of cloud (H1)" from a variable of type double */ inline void set_cloud_h1(double val, int conf=-1) { setd(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 13), val, conf); } /** Set the value of "Height of base of cloud (H1)" from a wreport::Var */ inline void set_cloud_h1_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 13), val); } /** Get the "Height of base of cloud (H1)" physical value stored in the message */ inline const wreport::Var* get_cloud_h1_var() const { return get(Level(256,MISSING_INT,259,1), Trange(254,0,0), WR_VAR(0, 20, 13)); } /** Set the value of "Cloud amount (N2)" from a variable of type int */ inline void set_cloud_n2(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 11), val, conf); } /** Set the value of "Cloud amount (N2)" from a wreport::Var */ inline void set_cloud_n2_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 11), val); } /** Get the "Cloud amount (N2)" physical value stored in the message */ inline const wreport::Var* get_cloud_n2_var() const { return get(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 11)); } /** Set the value of "Cloud amount (C2)" from a variable of type int */ inline void set_cloud_c2(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud amount (C2)" from a wreport::Var */ inline void set_cloud_c2_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud amount (C2)" physical value stored in the message */ inline const wreport::Var* get_cloud_c2_var() const { return get(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Height of base of cloud (H2)" from a variable of type double */ inline void set_cloud_h2(double val, int conf=-1) { setd(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 13), val, conf); } /** Set the value of "Height of base of cloud (H2)" from a wreport::Var */ inline void set_cloud_h2_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 13), val); } /** Get the "Height of base of cloud (H2)" physical value stored in the message */ inline const wreport::Var* get_cloud_h2_var() const { return get(Level(256,MISSING_INT,259,2), Trange(254,0,0), WR_VAR(0, 20, 13)); } /** Set the value of "Cloud amount (N3)" from a variable of type int */ inline void set_cloud_n3(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 11), val, conf); } /** Set the value of "Cloud amount (N3)" from a wreport::Var */ inline void set_cloud_n3_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 11), val); } /** Get the "Cloud amount (N3)" physical value stored in the message */ inline const wreport::Var* get_cloud_n3_var() const { return get(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 11)); } /** Set the value of "Cloud amount (C3)" from a variable of type int */ inline void set_cloud_c3(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud amount (C3)" from a wreport::Var */ inline void set_cloud_c3_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud amount (C3)" physical value stored in the message */ inline const wreport::Var* get_cloud_c3_var() const { return get(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Height of base of cloud (H3)" from a variable of type double */ inline void set_cloud_h3(double val, int conf=-1) { setd(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 13), val, conf); } /** Set the value of "Height of base of cloud (H3)" from a wreport::Var */ inline void set_cloud_h3_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 13), val); } /** Get the "Height of base of cloud (H3)" physical value stored in the message */ inline const wreport::Var* get_cloud_h3_var() const { return get(Level(256,MISSING_INT,259,3), Trange(254,0,0), WR_VAR(0, 20, 13)); } /** Set the value of "Cloud amount (N4)" from a variable of type int */ inline void set_cloud_n4(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 11), val, conf); } /** Set the value of "Cloud amount (N4)" from a wreport::Var */ inline void set_cloud_n4_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 11), val); } /** Get the "Cloud amount (N4)" physical value stored in the message */ inline const wreport::Var* get_cloud_n4_var() const { return get(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 11)); } /** Set the value of "Cloud amount (C4)" from a variable of type int */ inline void set_cloud_c4(int val, int conf=-1) { seti(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 12), val, conf); } /** Set the value of "Cloud amount (C4)" from a wreport::Var */ inline void set_cloud_c4_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 12), val); } /** Get the "Cloud amount (C4)" physical value stored in the message */ inline const wreport::Var* get_cloud_c4_var() const { return get(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 12)); } /** Set the value of "Height of base of cloud (H4)" from a variable of type double */ inline void set_cloud_h4(double val, int conf=-1) { setd(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 13), val, conf); } /** Set the value of "Height of base of cloud (H4)" from a wreport::Var */ inline void set_cloud_h4_var(const wreport::Var& val) { set(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 13), val); } /** Get the "Height of base of cloud (H4)" physical value stored in the message */ inline const wreport::Var* get_cloud_h4_var() const { return get(Level(256,MISSING_INT,259,4), Trange(254,0,0), WR_VAR(0, 20, 13)); } dballe-8.6/dballe/msg/wr_exporters/0000755000175000017500000000000013602152021014357 500000000000000dballe-8.6/dballe/msg/wr_exporters/flight.cc0000644000175000017500000004251413554564112016107 00000000000000#include "dballe/msg/wr_codec.h" #include "dballe/core/shortcuts.h" #include "dballe/msg/msg.h" #include "dballe/msg/context.h" #include using namespace wreport; using namespace std; #define AIREP_NAME "airep" #define AIREP_DESC "AIREP (autodetect)" #define AIREP_ECMWF_NAME "airep-ecmwf" #define AIREP_ECMWF_DESC "AIREP ECMWF (4.142)" #define AMDAR_NAME "amdar" #define AMDAR_DESC "AMDAR (autodetect)" #define AMDAR_ECMWF_NAME "amdar-ecmwf" #define AMDAR_ECMWF_DESC "AMDAR ECMWF (4.144)" #define AMDAR_WMO_NAME "amdar-wmo" #define AMDAR_WMO_DESC "AMDAR WMO" #define ACARS_NAME "acars" #define ACARS_DESC "ACARS (autodetect)" #define ACARS_ECMWF_NAME "acars-ecmwf" #define ACARS_ECMWF_DESC "ACARS ECMWF (4.145)" #define ACARS_WMO_NAME "acars-wmo" #define ACARS_WMO_DESC "ACARS WMO" namespace dballe { namespace impl { namespace msg { namespace wr { namespace { // Base template for flights struct FlightBase : public Template { bool is_crex; const msg::Context* flight_ctx; FlightBase(const dballe::ExporterOptions& opts, const Messages& msgs) : Template(opts, msgs), flight_ctx(0) {} void add(wreport::Varcode code, const wreport::Var* var) const { Template::add(code, var); } void add(Varcode code, const Shortcut& shortcut) const { const Var* var = msg->get(shortcut); if (var) subset->store_variable(code, *var); else subset->store_variable_undef(code); } void add(Varcode code) const { add(code, code); } void add(Varcode code, Varcode srccode) const { const Var* var = flight_ctx->values.maybe_var(srccode); if (var) subset->store_variable(code, *var); else subset->store_variable_undef(code); } void setupBulletin(wreport::Bulletin& bulletin) override { Template::setupBulletin(bulletin); // Use old table for old templates if (BufrBulletin* b = dynamic_cast(&bulletin)) { b->master_table_version_number = 13; } is_crex = dynamic_cast(&bulletin) != 0; bulletin.data_category = 4; bulletin.data_subcategory = 255; } void to_subset(const Message& msg, wreport::Subset& subset) override { Template::to_subset(msg, subset); // Find what is the level where the airplane is in flight_ctx = 0; for (const auto& ctx : msg.data) { if (ctx.trange != Trange::instant()) continue; bool use = false; switch (ctx.level.ltype1) { case 100: use = ctx.values.maybe_var(sc::press.code) != nullptr || ctx.values.maybe_var(sc::height_station.code) != nullptr; break; case 102: use = ctx.values.maybe_var(sc::height_station.code) != nullptr; break; } if (use) { if (flight_ctx != 0) error_consistency::throwf("contradicting height indication found (both %d and %d)", flight_ctx->level.ltype1, ctx.level.ltype1); flight_ctx = &ctx; } } if (flight_ctx == 0) throw error_notfound("no airplane pressure or height found in flight message"); } }; struct Airep : public FlightBase { Airep(const dballe::ExporterOptions& opts, const Messages& msgs) : FlightBase(opts, msgs) {} virtual const char* name() const { return AIREP_NAME; } virtual const char* description() const { return AIREP_DESC; } virtual void setupBulletin(wreport::Bulletin& bulletin) { FlightBase::setupBulletin(bulletin); bulletin.data_subcategory_local = 142; // Data descriptor section bulletin.datadesc.clear(); bulletin.datadesc.push_back(WR_VAR(3, 11, 1)); if (!is_crex) { bulletin.datadesc.push_back(WR_VAR(2, 22, 0)); bulletin.datadesc.push_back(WR_VAR(1, 1, 18)); bulletin.datadesc.push_back(WR_VAR(0, 31, 31)); bulletin.datadesc.push_back(WR_VAR(0, 1, 31)); bulletin.datadesc.push_back(WR_VAR(0, 1, 32)); bulletin.datadesc.push_back(WR_VAR(1, 1, 18)); bulletin.datadesc.push_back(WR_VAR(0, 33, 7)); } bulletin.load_tables(); } void to_subset(const Message& msg, wreport::Subset& subset) override { FlightBase::to_subset(msg, subset); /* 0 */ add(WR_VAR(0, 1, 6), sc::ident); /* 1 */ add(WR_VAR(0, 2, 61)); do_D01011(); do_D01012(); /* 7 */ add(WR_VAR(0, 5, 1), sc::latitude); /* 8 */ add(WR_VAR(0, 6, 1), sc::longitude); /* 9 */ add(WR_VAR(0, 8, 4)); /* 10 */ add(WR_VAR(0, 7, 2), WR_VAR(0, 7, 30)); /* HEIGHT OF STATION -> HEIGHT OR ALTITUDE */ /* 11 */ add(WR_VAR(0, 12, 1), WR_VAR(0, 12, 101)); /* TEMPERATURE/DRY-BULB TEMPERATURE */ /* 12 */ add(WR_VAR(0, 11, 1)); /* WIND DIRECTION */ /* 13 */ add(WR_VAR(0, 11, 2)); /* WIND SPEED */ /* 14 */ add(WR_VAR(0, 11, 31)); /* DEGREE OF TURBULENCE */ /* 15 */ add(WR_VAR(0, 11, 32)); /* HEIGHT OF BASE OF TURBULENCE */ /* 16 */ add(WR_VAR(0, 11, 33)); /* HEIGHT OF TOP OF TURBULENCE */ /* 17 */ add(WR_VAR(0, 20, 41)); /* AIRFRAME ICING */ if (!is_crex) { subset.append_fixed_dpb(WR_VAR(2, 22, 0), 18); if (opts.centre != MISSING_INT) subset.store_variable_i(WR_VAR(0, 1, 31), opts.centre); else subset.store_variable_undef(WR_VAR(0, 1, 31)); if (opts.application != MISSING_INT) subset.store_variable_i(WR_VAR(0, 1, 32), opts.application); else subset.store_variable_undef(WR_VAR(0, 1, 32)); } } }; struct Amdar : public Airep { Amdar(const dballe::ExporterOptions& opts, const Messages& msgs) : Airep(opts, msgs) {} virtual const char* name() const { return AMDAR_NAME; } virtual const char* description() const { return AMDAR_DESC; } virtual void setupBulletin(wreport::Bulletin& bulletin) { Airep::setupBulletin(bulletin); bulletin.data_subcategory_local = 144; } }; struct AmdarWMO : public FlightBase { AmdarWMO(const dballe::ExporterOptions& opts, const Messages& msgs) : FlightBase(opts, msgs) {} virtual const char* name() const { return AMDAR_WMO_NAME; } virtual const char* description() const { return AMDAR_WMO_DESC; } virtual void setupBulletin(wreport::Bulletin& bulletin) { FlightBase::setupBulletin(bulletin); bulletin.data_subcategory = 255; bulletin.data_subcategory_local = 144; // Data descriptor section bulletin.datadesc.clear(); //bulletin.datadesc.push_back(WR_VAR(0, 1, 33)); // 0 //bulletin.datadesc.push_back(WR_VAR(0, 1, 34)); // 1 bulletin.datadesc.push_back(WR_VAR(3, 11, 5)); // 2 bulletin.datadesc.push_back(WR_VAR(0, 8, 4)); // 20 bulletin.datadesc.push_back(WR_VAR(0, 2, 64)); // 21 bulletin.datadesc.push_back(WR_VAR(0, 13, 3)); // 22 bulletin.datadesc.push_back(WR_VAR(0, 12, 103)); // 23 bulletin.datadesc.push_back(WR_VAR(0, 13, 2)); // 24 bulletin.datadesc.push_back(WR_VAR(1, 2, 0)); bulletin.datadesc.push_back(WR_VAR(0, 31, 1)); // 25 bulletin.datadesc.push_back(WR_VAR(0, 11, 75)); bulletin.datadesc.push_back(WR_VAR(0, 11, 76)); bulletin.datadesc.push_back(WR_VAR(0, 11, 37)); bulletin.datadesc.push_back(WR_VAR(0, 11, 39)); bulletin.datadesc.push_back(WR_VAR(0, 11, 77)); bulletin.datadesc.push_back(WR_VAR(0, 20, 42)); bulletin.datadesc.push_back(WR_VAR(0, 20, 43)); bulletin.datadesc.push_back(WR_VAR(0, 20, 44)); bulletin.datadesc.push_back(WR_VAR(0, 20, 45)); bulletin.datadesc.push_back(WR_VAR(0, 20, 41)); bulletin.datadesc.push_back(WR_VAR(0, 2, 5)); bulletin.datadesc.push_back(WR_VAR(0, 2, 62)); bulletin.datadesc.push_back(WR_VAR(0, 2, 70)); bulletin.datadesc.push_back(WR_VAR(0, 2, 65)); bulletin.datadesc.push_back(WR_VAR(0, 7, 4)); bulletin.datadesc.push_back(WR_VAR(0, 33, 26)); bulletin.load_tables(); } void to_subset(const Message& msg, wreport::Subset& subset) override { FlightBase::to_subset(msg, subset); Level lev; ///* 0 */ add(WR_VAR(0, 1, 33)); ///* 1 */ add(WR_VAR(0, 1, 34)); /* 2 */ add(WR_VAR(0, 1, 8), sc::ident); /* 3 */ add(WR_VAR(0, 1, 23)); /* 4 */ add(WR_VAR(0, 5, 1), sc::latitude); /* 5 */ add(WR_VAR(0, 6, 1), sc::longitude); do_D01011(); do_D01013(); /* 12 */ if (const wreport::Var* v = flight_ctx->values.maybe_var(WR_VAR(0, 7, 30))) add(WR_VAR(0, 7, 10), v); else if (flight_ctx->level.ltype1 == 102) subset.store_variable_d(WR_VAR(0, 7, 10), (double)flight_ctx->level.l1 / 1000.0); else subset.store_variable_undef(WR_VAR(0, 7, 10)); /* 13 */ add(WR_VAR(0, 8, 9)); /* 14 */ add(WR_VAR(0, 11, 1)); /* 15 */ add(WR_VAR(0, 11, 2)); /* 16 */ add(WR_VAR(0, 11, 31)); /* 17 */ add(WR_VAR(0, 11, 36)); /* 18 */ add(WR_VAR(0, 12,101)); /* 19 */ add(WR_VAR(0, 33, 25)); /* 20 */ add(WR_VAR(0, 8, 4)); /* 21 */ add(WR_VAR(0, 2, 64)); /* 22 */ add(WR_VAR(0, 13, 3)); /* 23 */ add(WR_VAR(0, 12,103)); /* 24 */ add(WR_VAR(0, 13, 2)); /* 25 */ subset.store_variable_i(WR_VAR(0, 31, 1), 0); // FIXME: no replicated section so far //102000 replicate 2 descriptors (delayed 031001) times // 011075 MEAN TURBULENCE INTENSITY (EDDY DISSIPATION RATE)[M**(2/3)/S] // 011076 PEAK TURBULENCE INTENSITY (EDDY DISSIPATION RATE)[M**(2/3)/S] add(WR_VAR(0, 11, 37)); add(WR_VAR(0, 11, 39)); add(WR_VAR(0, 11, 77)); add(WR_VAR(0, 20, 42)); add(WR_VAR(0, 20, 43)); add(WR_VAR(0, 20, 44)); add(WR_VAR(0, 20, 45)); add(WR_VAR(0, 20, 41)); add(WR_VAR(0, 2, 5)); add(WR_VAR(0, 2, 62)); add(WR_VAR(0, 2, 70)); add(WR_VAR(0, 2, 65)); add(WR_VAR(0, 7, 4)); add(WR_VAR(0, 33, 26)); } }; struct Acars : public FlightBase { Acars(const dballe::ExporterOptions& opts, const Messages& msgs) : FlightBase(opts, msgs) {} virtual const char* name() const { return ACARS_NAME; } virtual const char* description() const { return ACARS_DESC; } virtual void setupBulletin(wreport::Bulletin& bulletin) { FlightBase::setupBulletin(bulletin); bulletin.data_subcategory_local = 145; // Data descriptor section bulletin.datadesc.clear(); bulletin.datadesc.push_back(WR_VAR(0, 1, 6)); bulletin.datadesc.push_back(WR_VAR(0, 1, 8)); bulletin.datadesc.push_back(WR_VAR(0, 2, 61)); bulletin.datadesc.push_back(WR_VAR(0, 2, 62)); bulletin.datadesc.push_back(WR_VAR(0, 2, 2)); bulletin.datadesc.push_back(WR_VAR(0, 2, 5)); bulletin.datadesc.push_back(WR_VAR(0, 2, 70)); bulletin.datadesc.push_back(WR_VAR(0, 2, 63)); bulletin.datadesc.push_back(WR_VAR(0, 2, 1)); bulletin.datadesc.push_back(WR_VAR(3, 1, 11)); bulletin.datadesc.push_back(WR_VAR(3, 1, 12)); bulletin.datadesc.push_back(WR_VAR(3, 1, 23)); bulletin.datadesc.push_back(WR_VAR(0, 8, 4)); bulletin.datadesc.push_back(WR_VAR(0, 7, 4)); bulletin.datadesc.push_back(WR_VAR(0, 8, 21)); bulletin.datadesc.push_back(WR_VAR(0, 11, 1)); bulletin.datadesc.push_back(WR_VAR(0, 11, 2)); bulletin.datadesc.push_back(WR_VAR(0, 11, 31)); bulletin.datadesc.push_back(WR_VAR(0, 11, 34)); bulletin.datadesc.push_back(WR_VAR(0, 11, 35)); bulletin.datadesc.push_back(WR_VAR(0, 12, 1)); bulletin.datadesc.push_back(WR_VAR(0, 12, 3)); bulletin.datadesc.push_back(WR_VAR(0, 13, 3)); bulletin.datadesc.push_back(WR_VAR(0, 20, 41)); if (!is_crex) { bulletin.datadesc.push_back(WR_VAR(2, 22, 0)); bulletin.datadesc.push_back(WR_VAR(1, 1, 28)); bulletin.datadesc.push_back(WR_VAR(0, 31, 31)); bulletin.datadesc.push_back(WR_VAR(0, 1, 31)); bulletin.datadesc.push_back(WR_VAR(0, 1, 201)); bulletin.datadesc.push_back(WR_VAR(1, 1, 28)); bulletin.datadesc.push_back(WR_VAR(0, 33, 7)); } bulletin.load_tables(); } void to_subset(const Message& msg, wreport::Subset& subset) override { FlightBase::to_subset(msg, subset); /* 0 */ add(WR_VAR(0, 1, 6)); /* 1 */ add(WR_VAR(0, 1, 8), sc::ident); /* 2 */ add(WR_VAR(0, 2, 61)); /* 3 */ add(WR_VAR(0, 2, 62)); /* 4 */ add(WR_VAR(0, 2, 2)); /* 5 */ add(WR_VAR(0, 2, 5)); /* 6 */ add(WR_VAR(0, 2, 70)); /* 7 */ add(WR_VAR(0, 2, 63)); /* 8 */ add(WR_VAR(0, 2, 1)); do_D01011(); do_D01012(); /* 14 */ add(WR_VAR(0, 5, 2), sc::latitude); /* 15 */ add(WR_VAR(0, 6, 2), sc::longitude); /* 16 */ add(WR_VAR(0, 8, 4)); /* 17 */ add(WR_VAR(0, 7, 4), WR_VAR(0, 10, 4)); /* 18 */ add(WR_VAR(0, 8, 21)); /* 19 */ add(WR_VAR(0, 11, 1)); /* WIND DIRECTION */ /* 20 */ add(WR_VAR(0, 11, 2)); /* WIND SPEED */ /* 21 */ add(WR_VAR(0, 11, 31)); /* DEGREE OF TURBULENCE */ /* 22 */ add(WR_VAR(0, 11, 34)); /* VERTICAL GUST VELOCITY */ /* 23 */ add(WR_VAR(0, 11, 35)); /* VERTICAL GUST ACCELERATION */ /* 24 */ add(WR_VAR(0, 12, 1), WR_VAR(0, 12, 101)); /* TEMPERATURE/DRY-BULB TEMPERATURE */ /* 25 */ add(WR_VAR(0, 12, 3), WR_VAR(0, 12, 103)); /* DEW-POINT TEMPERATURE */ /* 26 */ add(WR_VAR(0, 13, 3)); /* RELATIVE HUMIDITY */ /* 27 */ add(WR_VAR(0, 20, 41)); /* AIRFRAME ICING */ if (!is_crex) { subset.append_fixed_dpb(WR_VAR(2, 22, 0), 28); if (opts.centre != MISSING_INT) subset.store_variable_i(WR_VAR(0, 1, 31), opts.centre); else subset.store_variable_undef(WR_VAR(0, 1, 31)); if (opts.application != MISSING_INT) subset.store_variable_i(WR_VAR(0, 1, 201), opts.application); else subset.store_variable_undef(WR_VAR(0, 1, 201)); } } }; struct AcarsWMO : public AmdarWMO { AcarsWMO(const dballe::ExporterOptions& opts, const Messages& msgs) : AmdarWMO(opts, msgs) {} virtual const char* name() const { return ACARS_WMO_NAME; } virtual const char* description() const { return ACARS_WMO_DESC; } virtual void setupBulletin(wreport::Bulletin& bulletin) { AmdarWMO::setupBulletin(bulletin); bulletin.data_subcategory = 255; bulletin.data_subcategory_local = 145; } }; } // anonymous namespace void register_flight(TemplateRegistry& r) { r.register_factory(4, AIREP_NAME, AIREP_DESC, [](const dballe::ExporterOptions& opts, const Messages& msgs) { return unique_ptr