pax_global_header00006660000000000000000000000064130662627710014524gustar00rootroot0000000000000052 comment=0fa793dd75beb3d8927cbcb20ea0cbb93c0c71dd cl-csv-20170403-git/000077500000000000000000000000001306626277100137365ustar00rootroot00000000000000cl-csv-20170403-git/.gitignore000066400000000000000000000000511306626277100157220ustar00rootroot00000000000000.svn build dist *~ *# *.fasl .git _darcs cl-csv-20170403-git/DOCUMENTATION.md000066400000000000000000000120521306626277100162710ustar00rootroot00000000000000 CL-CSV - a Common Lisp library for csv reading and writing ===== # Abstract A common lisp library providing easy csv reading and writing. The code license is found [here](https://github.com/AccelerationNet/cl-csv/blob/master/LICENSE). # Contents 1. [Download](#download) 2. [The CL-CSV dictionary](#dictionary) 1. [`*default-external-format*`](#*default-external-format*) 2. [`*newline*`](#*newline*) 3. [`*quote*`](#*quote*) 4. [`*quote-escape*`](#*quote-escape*) 5. [`*separator*`](#*separator*) 6. [`csv-parse-error`](#csv-parse-error) 7. [`csv-parse-error`](#csv-parse-error) 8. [`do-csv`](#do-csv) 9. [`format-csv-value`](#format-csv-value) 10. [`read-csv`](#read-csv) 11. [`read-csv-row`](#read-csv-row) 12. [`write-csv`](#write-csv) 13. [`write-csv-row`](#write-csv-row) 14. [`write-csv-value`](#write-csv-value) 3. [Acknowledgements](#ack) # Download CL-CSV together with this documentation can be downloaded from [https://github.com/AccelerationNet/cl-csv](https://github.com/AccelerationNet/cl-csv). # The CL-CSV dictionary _[Special variable]_ **\*default-external-format*** The external format used for opening files _[Special variable]_ **\*newline*** Default newline string _[Special variable]_ **\*quote*** Default quote character _[Special variable]_ **\*quote-escape*** Default setting for escaping quotes _[Special variable]_ **\*separator*** Default separator character - - - _[Condition type]_ **csv-parse-error** - - - _[Function]_ **csv-parse-error** ( *msg `&rest` args* ) =\> \*result* - - - _[Macro]_ **do-csv** *((row-var stream-or-pathname `&rest` read-csv-keys) `&body` body)* =\> \*result* row-var: a variable that is passed into _body_ stream-or-pathname: a stream or a pathname to read the CSV data from read-csv-keys: keys and values passed to the _read-csv_ function body: body of the macro - - - _[Generic function]_ **format-csv-value** ( *val* ) =\> \*result* - - - _[Method]_ **format-csv-value** ( *val* ) =\> \*result* Print values in ways that are most cross compatible with the csv format - - - _[Function]_ **read-csv** ( *stream-or-string* `&key` *row-fn* *map-fn* *sample* *skip-first-p* ( separator \*separator\* ) (quote \*quote\*) (escape \*quote-escape\*))* =\> \*result* Read in a CSV by data-row (which due to quoted newlines may be more than one line from the stream) row-fn: passing this parameter will cause this read to be streaming and results will be discarded after the row-fn is called with data map-fn: used for manipulating the data by row during collection if specified; (funcall map-fn data) is collected instead of data sample: when a positive integer, only take that many samples from the input file skip-first-p: when true, skips the first line in the csv _Keywords:_ separator: character separating between data cells. Defaults to \*separator* quote: quoting character for text strings. Defaults to \*quote* escape: escape character. Defaults to \*quote-escape* - - - _[Function]_ **read-csv-row** ( *stream-or-string* `&key` (separator \*separator\*) (quote \*quote\*) (escape \*quote-escape\*) `&aux` *current state line llen c elen*) =\> \*result\* Read in a CSV by _data-row_ (which due to quoted newlines may be more than one line from the stream) - - - _[Function]_ **write-csv** (*rows-of-items* `&key` *stream* (separator \*separator\*) (quote \*quote\*) (escape \*quote-escape\*) (newline \*newline\*) (always-quote \*always-quote\*) ) =\> \*result\* Writes a CSV rows-of-items: iterable _Keywords:_ stream: stream to write to. Default: nil. quote: quoting character. Defaults to \*quote\* escape: escaping character. Defaults to \*quote-escape\* newline: newline character. Defaults to \*newline\* always-quote: Defaults to \*always-quote\* - - - _[Function]_ **write-csv-row** ( *items* `&key` *stream* (separator \*separator\*) (quote \*quote\*) (escape \*quote-escape\*) (newline \*newline\*) (always-quote \*always-quote\*) ) =\> \*result* Writes a list items to stream rows-of-items: iterable _Keywords:_ stream: stream to write to. Default: nil. quote: quoting character. Defaults to \*quote\* escape: escaping character. Defaults to \*quote-escape\* newline: newline character. Defaults to \*newline\* always-quote: Defaults to \*always-quote\* - - - _[Generic function]_ **write-csv-value** ( *val csv-stream* `&key` *formatter quote separator escape always-quote* ) =\> \*result* - - - _[Method]_ **write-csv-value** ( *val csv-stream* `&key` *formatter quote separator escape always-quote* ) =\> \*result* Writes val to csv-stream in a formatted fashion. _Keywords:_ formatter: used to format val. Defaults to format-csv-value. quote: quoting character. Defaults to \*quote* escape: escaping character. Defaults to \*quote-escape* newline: newline character. Defaults to \*newline* always-quote: Defaults to \*always-quote* # Acknowledgements This documentation was prepared with [DOCUMENTATION-TEMPLATE](http://weitz.de/documentation-template/), then passed through [pandoc](http://johnmacfarlane.net/pandoc/index.html), *then* hand-edited. cl-csv-20170403-git/LICENSE000066400000000000000000000032301306626277100147410ustar00rootroot00000000000000;; Copyright (c) 2011 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; Copyright (c) 2002-2006, Edward Marco Baringer ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cl-csv-20170403-git/README.md000066400000000000000000000117521306626277100152230ustar00rootroot00000000000000# cl-csv This library aims to simplify working with csvs to the bare minimum of tedium * reads/writes csvs from/to strings, streams and files * support streaming reads (allowing processing very large csvs, through read-csv's row-fn paramter) * supports custom data formating * settable quote, separator and quote-escapes * supports multiline quoted data * A test suite ## Rationale I had many scattered, not well tested, not easily runable pieces of csv code. I was unhappy with this situation then decided to refactor all of this into a single location. I wrote tests for it and had a library so I thought I might release it. This project started as extensions and bugfixes on arnesi's CSV. I then looked around and saw there are other csv libs out there that probably mostly accomplished what I had set out to do. However, I already had code that was tested and had an easier license (BSD) so, I figured why not just release it anyway. ### Other Available CSV libs * http://members.optusnet.com.au/apicard/csv-parser.lisp * http://www.cliki.net/fare-csv ## Escaping and quotes There are two modes for escaping currently * :quote - by default cl-csv treats `""` as an escape for a single double-quote * :following - read the character after the escape sequence verbatim, commonly the `*quote-escape*` will be set to `#\\` when the escape mode is following. ## Signals and Restarts * `*enable-signals*` will cause a csv-data-read or csv-row-read to be signaled for each piece of data and row read. There is a `filter` restart available which will cause the filter value to be used instead. Enabling signals is ~2xs as slow as not, so by default they are not enabled. * `in-csv` iterate clause and `read-csv` support `continue` and `filter` restarts for errors occuring during read-csv-row ## Library Integration * [data-table](https://github.com/AccelerationNet/data-table) - functions for building data-tables from csv's, must `(asdf:load-system :cl-csv-data-table)` * [clsql](http://clsql.b9.com/) must `(asdf:load-system :cl-csv-clsql)` * import-from-csv * serial-import-from-csv * [iterate](http://common-lisp.net/project/iterate) - provides an `in-csv` driver clause for iterating over a CSV ## Examples ```lisp ;; read a file into a list of lists (cl-csv:read-csv #P"file.csv") => (("1" "2" "3") ("4" "5" "6")) ;; read a file that's tab delimited (cl-csv:read-csv #P"file.tab" :separator #\Tab) ;; read a file and return a list of objects created from each row (cl-csv:read-csv #P"file.csv" :map-fn #'(lambda (row) (make-instance 'object :foo (nth 0 row) :baz (nth 2 row)))) ;; read csv from a string (streams also supported) (cl-csv:read-csv "1,2,3 4,5,6") => (("1" "2" "3") ("4" "5" "6")) ;; loop over a CSV for effect (let ((sum 0)) (cl-csv:do-csv (row #P"file.csv") (incf sum (parse-integer (nth 0 row)))) sum) ;; loop over a CSV using iterate (iter (for (foo bar baz) in-csv #P"file.csv") (collect (make-instance 'object :foo foo :baz baz))) ``` ## Authors * [Acceleration.net](http://www.acceleration.net/) * [Russ Tyndall](http://russ.unwashedmeme.com/blog) * [Nathan Bird](http://the.unwashedmeme.com/blog) * [Ryan Davis](http://ryepup.unwashedmeme.com/blog) ``` ;; Copyright (c) 2011 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; Copyright (c) 2002-2006, Edward Marco Baringer ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. ``` cl-csv-20170403-git/cl-csv-clsql.asd000066400000000000000000000042431306626277100167350ustar00rootroot00000000000000;; -*- lisp -*- (eval-when (:compile-toplevel :load-toplevel :execute) (unless (find-package :cl-csv.system) (defpackage :cl-csv.system (:use :common-lisp :asdf)))) (in-package cl-csv.system) (defsystem :cl-csv-clsql :author "Russ Tyndall (russ@acceleration.net), Acceleration.net" :description "Facilities for reading and writing CSV format files (and importing and exporting csvs from databases)" :licence "BSD" :version "1.0" :components ((:file "data-table") (:file "clsql")) :depends-on (:clsql-helper :cl-csv :data-table-clsql)) ;; Copyright (c) 2011 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cl-csv-20170403-git/cl-csv-data-table.asd000066400000000000000000000036531306626277100176210ustar00rootroot00000000000000 (asdf:defsystem :cl-csv-data-table :author "Russ Tyndall (russ@acceleration.net), Acceleration.net" :description "Facilities for converting CSV data to data-tables" :licence "BSD" :version "1.0" :components ((:file "data-table")) :depends-on (:cl-csv :data-table)) ;; Copyright (c) 2012 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; -- This file contributed by Cyrus Harmon ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cl-csv-20170403-git/cl-csv.asd000066400000000000000000000054211306626277100156200ustar00rootroot00000000000000;; -*- lisp -*- (eval-when (:compile-toplevel :load-toplevel :execute) (unless (find-package :cl-csv.system) (defpackage :cl-csv.system (:use :common-lisp :asdf)))) (in-package cl-csv.system) (defsystem :cl-csv :author "Russ Tyndall (russ@acceleration.net), Acceleration.net" :description "Facilities for reading and writing CSV format files" :licence "BSD" :version "1.0" :serial t :components ((:file "packages") (:file "vars") (:file "read-until") (:file "csv")) :depends-on (:iterate :alexandria :cl-interpol)) (defsystem :cl-csv-test :author "Russ Tyndall (russ@acceleration.net), Acceleration.net" :description "Tests for a library providing a cl-csv class, and useful functionality around this" :licence "BSD" :version "1.0" :components ((:module :tests :serial t :components ((:file "csv")))) :depends-on (:cl-csv :lisp-unit2)) (defmethod asdf:perform ((o asdf:test-op) (c (eql (find-system :cl-csv)))) (asdf:oos 'asdf:load-op :cl-csv-test) (let ((*package* (find-package :cl-csv-test))) (eval (read-from-string "(run-all-tests)")))) ;; Copyright (c) 2011 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; Copyright (c) 2002-2006, Edward Marco Baringer ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cl-csv-20170403-git/clsql.lisp000066400000000000000000000137251306626277100157550ustar00rootroot00000000000000(in-package :cl-csv) (cl-interpol:enable-interpol-syntax) (defun exec (sql) (clsql-sys:execute-command sql)) (defmethod format-csv-value ((val clsql-sys:date)) (clsql-helper:print-nullable-date val)) (defmethod format-csv-value ((val clsql-sys:wall-time)) (clsql-helper:print-nullable-datetime val)) (defun export-query ( sql &key stream path) (with-csv-output-stream (s (or stream path)) (multiple-value-bind (rows cols) (clsql:query sql :flatp t) (write-csv-row cols :stream s) (write-csv rows :stream s)))) (defun import-from-csv (table-name &rest keys &key file data-table (sample-size 1000) schema (should-have-serial-id "id") excluded-columns row-fn (log-fn #'(lambda (&rest args) (declare (ignore args)))) (log-frequency 1000) &aux (cl-interpol:*list-delimiter* ",") (*print-pretty* nil)) "Will make a best effor to create a table matching the csv's schema and then row-fn (data-row schema table columns) allows you to take actions on a row (before insert). returning false will prevent the default insert log-fn (msg &rest args) function to log progress log-frequency : how frequent (measured in rows) to log progress " ;;; TODO: accept column names from params ;;; TODO: figure out how to process files without columns in the first row ;;; TODO: figure out how to override type guesses ;;; TODO: check if the table already exists and skip the guessing (funcall log-fn "Starting import ~a" table-name) (let ((dt (or data-table (get-data-table-from-csv file t t sample-size))) (keys (copy-list keys))) ;; we are putting this in a database, we cannot have empty column names (setf (data-table:column-names dt) (iter (with i = 0) (for c in (data-table:column-names dt)) (collect (or c #?"anon_${ (incf i) }")))) (dolist (k '(:file :data-table :row-fn :sample-size :log-fn :log-frequency)) (remf keys k)) (funcall log-fn "CSV scanned for type information") (when (and should-have-serial-id (member should-have-serial-id (data-table:column-names dt) :test #'string-equal)) (error #?"This table already has an id column name `${should-have-serial-id}` Column! Perhaps you wish to turn off should-have-serial-id or assign it a different name?")) (apply #'data-table:ensure-table-for-data-table dt table-name keys) (funcall log-fn "Created table, starting import") (let ((start-time (get-universal-time))) (flet ((log-progress (row-num &optional (msg "Processing row")) (let ((elapsed (- (get-universal-time) start-time))) (funcall log-fn "~a ~a. ~ds elapsed (~,2f rows/sec) " msg row-num elapsed (if (zerop elapsed) "Inf" (/ row-num elapsed)))))) (iter (with importer = (data-table::make-row-importer dt table-name :schema schema :excluded-columns excluded-columns :row-fn row-fn)) (for row in-csv file SKIPPING-HEADER T) (for row-num from 1) (funcall importer row) (when (zerop (mod row-num log-frequency)) (log-progress row-num)) (finally (log-progress row-num "Finished, total processed: "))))))) (defun serial-import-from-csv (table-name &key file (column-names :first-row) (schema "public") (column-transform #'data-table::english->postgres) (progress-stream t) (progress-mod 5000) (data-munger (lambda (row) (mapcar #'clsql-helper:format-value-for-database row))) (log (lambda (msg &rest args) (apply #'format progress-stream msg args))) (on-error 'continue-importing) &aux columns (cl-interpol:*list-delimiter* ", ")) "Will make a best effor to create a table matching the csv's schema and then data-munger : a function that changes the the data row to be inserted (for conversion to other types etc) progress-stream: the default log stream to print to progress-mod: how often we should report progress (in number of rows) log: is a function that accepts msg and args to display status updates to the user on-error: is a restart to run when an error is experienced, if nil, it will simply allow the error to go up the stack. the default is to continue with the import, skipping the row that caused errors " (unless (eql column-names :first-row) (setf columns (data-table::sql-escaped-column-names column-names :transform column-transform))) (iter (for row in-csv file) (for cnt from 0) (if (and (eql column-names :first-row) (first-iteration-p)) (setf columns (data-table::sql-escaped-column-names row :transform column-transform)) (restart-case (handler-bind ((error (lambda (c) (funcall log "Error importing ROW ~D of file: ~S~%~S~%~A~%~S" cnt file row c c) (when on-error (when (find-restart on-error) (invoke-restart on-error)))))) (exec #?"INSERT INTO ${schema}.${table-name} (@{ columns }) VALUES ( @{ (funcall data-munger row) } )") (when (zerop (mod cnt progress-mod)) (funcall log "Imported ~D rows~%" cnt ))) (continue-importing () :report "Continue Importing the file, skipping this row of data"))))) cl-csv-20170403-git/csv.lisp000066400000000000000000000643001306626277100154250ustar00rootroot00000000000000;; -*- lisp -*- (in-package :cl-csv) (cl-interpol:enable-interpol-syntax) ;;;; * Reading and Writing files in Comma-Seperated-Values format ;;;; Generating CSV files from lisp data (defun white-space? (c) (member c '(#\newline #\tab #\space #\return))) (define-condition csv-parse-error (error) ((format-control :accessor format-control :initarg :format-control :initform nil) (format-args :accessor format-args :initarg :format-args :initform nil)) (:report (lambda (c s &aux (ctrl (format-control c))) (typecase ctrl (condition (format s "CSV-PARSE-ERROR: internal-error ~A" ctrl)) (string (apply #'format s ctrl (format-args c))))))) (defun csv-parse-error (msg &rest args) (error 'csv-parse-error :format-control msg :format-args args)) (define-condition csv-data-read () ((data :accessor data :initarg :data :initform nil))) (defun csv-data-read ( data ) (if *enable-signals* (restart-case (progn (signal 'csv-data-read :data data ) data) (filter (new-data) new-data)) data)) (define-condition csv-row-read () ((row :accessor row :initarg :row :initform nil))) (defun csv-row-read ( row ) (if *enable-signals* (restart-case (progn (signal 'csv-row-read :row row ) row) (filter (new-row) new-row)) row)) ;;;; Writing csvs (defgeneric format-csv-value (val) (:documentation "Print values in ways that are most cross compatible with the csv format") (:method (val) (typecase val ((or float ratio) (format nil "~F" val)) (string val) (null "") (t (princ-to-string val))))) (defun %char-in (c to-check) (typecase to-check (character (char= c to-check)) (string (iter (for c2 in-string to-check) (thereis (char= c c2)))))) (defun chars-in (chars-to-check value-to-look-through) "returns true if any of the chars-to-check is found in the value-to-look-through" (iter (for c1 in-string value-to-look-through) (thereis (iter (for to-check in (alexandria:ensure-list chars-to-check)) (thereis (%char-in c1 to-check)))))) (defgeneric write-csv-value (val csv-stream &key formatter quote separator escape always-quote) (:documentation "Writes val to csv-stream in a formatted fashion. Keywords formatter: used to format val. Defaults to format-csv-value. quote: quoting character. Defaults to *quote* escape: escaping character. Defaults to *quote-escape* newline: newline character. Defaults to *write-newline* always-quote: Defaults to *always-quote*") (:method (val csv-stream &key (formatter #'format-csv-value) (quote *quote*) (separator *separator*) (escape *quote-escape*) (always-quote *always-quote*) (newline *write-newline*) &aux (formatted-value (funcall formatter val)) (should-quote (or always-quote (chars-in (list quote separator newline) formatted-value)))) (when should-quote (write-char quote csv-stream)) (iter (for char in-sequence formatted-value) (if (char= quote char) (write-sequence escape csv-stream) (write-char char csv-stream))) (when should-quote (write-char quote csv-stream)))) (defmacro with-csv-output-stream ((name inp) &body body) (alexandria:with-unique-names (opened?) `(multiple-value-bind (,name ,opened?) (%out-stream ,inp) (flet ((body () ,@body)) (unwind-protect (body) (when (and ,name ,opened?) (close ,name))))))) (defun %out-stream (stream-or-string) "creates a stream from the given thing, trying to DWIM" (etypecase stream-or-string (null (make-string-output-stream)) (stream stream-or-string) (pathname (values (open stream-or-string :direction :output :if-exists :supersede) t)))) (defun write-csv-row (items &key stream ((:separator *separator*) *separator*) ((:quote *quote*) *quote*) ((:escape *quote-escape*) *quote-escape*) ((:newline *write-newline*) *write-newline*) ((:always-quote *always-quote*) *always-quote*)) " Writes a list items to stream rows-of-items: iterable Keywords: stream: stream to write to. Default: nil. quote: quoting character. Defaults to *quote* escape: escaping character. Defaults to *quote-escape* newline: newline character. Defaults to *write-newline* always-quote: Defaults to *always-quote*" (with-csv-output-stream (csv-stream stream) (iter (for item in (alexandria:ensure-list items)) (unless (first-iteration-p) (write-char *separator* csv-stream)) (write-csv-value item csv-stream)) (write-sequence *write-newline* csv-stream) (unless stream (get-output-stream-string csv-stream)))) (defun write-csv (rows-of-items &key stream ((:separator *separator*) *separator*) ((:quote *quote*) *quote*) ((:escape *quote-escape*) *quote-escape*) ((:newline *write-newline*) *write-newline*) ((:always-quote *always-quote*) *always-quote*)) "Writes a csv to the given stream. rows-of-items: iterable Keywords: stream: stream to write to. Default: nil. nil - writes the rows to a string and returns it an open stream a pathname (overwrites if the file exists) quote: quoting character. Defaults to *quote* escape: escaping character. Defaults to *quote-escape* newline: newline character. Defaults to *write-newline* always-quote: Defaults to *always-quote*" (with-csv-output-stream (csv-stream stream) (iter (for row in rows-of-items) (write-csv-row row :stream csv-stream)) (unless stream (get-output-stream-string csv-stream)))) ;;;; Reading in CSV files (defun %escape-seq? (s i escape llen elen) (declare (type (or simple-string character) escape) (type string s) (type fixnum i llen elen)) (typecase escape (character (char= escape (schar s i))) (simple-string (when (<= (+ i elen) llen) (iter (declare (type fixnum eidx)) (with eidx = 0) (always (char= (char escape eidx) (char s (+ i eidx)))) (incf eidx) (while (< eidx elen))))))) (defvar *default-external-format* :default "the external format used for opening files") (defun %in-stream (stream-or-string) (typecase stream-or-string (string (make-string-input-stream stream-or-string)) (stream stream-or-string) (pathname (values (open stream-or-string :external-format *default-external-format*) T)))) (defmacro with-csv-input-stream ((name inp) &body body) (alexandria:with-unique-names (opened?) `(multiple-value-bind (,name ,opened?) (%in-stream ,inp) (flet ((body () ,@body)) (unwind-protect (body) (when (and ,name ,opened?) (close ,name))))))) (defun read-csv-row (stream-or-string &key ((:separator *separator*) *separator*) ((:quote *quote*) *quote*) ((:escape *quote-escape*) *quote-escape*) ((:unquoted-empty-string-is-nil *unquoted-empty-string-is-nil*) *unquoted-empty-string-is-nil*) ((:quoted-empty-string-is-nil *quoted-empty-string-is-nil*) *quoted-empty-string-is-nil*) ((:trim-outer-whitespace *trim-outer-whitespace*) *trim-outer-whitespace*) ((:newline *read-newline*) *read-newline*) ((:escape-mode *escape-mode*) *escape-mode*) &aux (*separator* (etypecase *separator* (string (if (= 1 (length *separator*)) (schar *separator* 0) (error "Only single character *separator* are currently supported:~A" *separator*))) (character *separator*))) (*read-newline* (etypecase *read-newline* (string (if (= 1 (length *read-newline*)) (schar *read-newline* 0) *read-newline*)) (character *read-newline*))) (current (make-array 20 :element-type 'character :adjustable t :fill-pointer 0)) (state :waiting) (i -1) (c #\null) (elen (etypecase *quote-escape* (string (length *quote-escape*)) (character 1))) items items-tail (nl-match -1) (nl-len (etypecase *read-newline* (string (length *read-newline*)) (character 1))) (nl-len-1 (- nl-len 1)) (use-read-line? (etypecase *read-newline* (character (char= *read-newline* #\newline)) (string nil))) newline-matched? (line (unless use-read-line? (make-array *buffer-size* :element-type 'character ))) (llen -1) read-line-got-a-newline?) "Read in a CSV by data-row (which due to quoted newlines may be more than one line from the stream) " ;; giant state machine parser ;; states: ;; waiting: we are between inputs, or have not started reading yet ;; collecting: collecting unquoted data ;; collecting-quoted: collecting quoted data ;; waiting-on-next: done collecting quoted data, now waitin for a ;; separator ;; this just ensures that a file opened here is closed here (with-csv-input-stream (in-stream stream-or-string) (csv-row-read (block results (loop do (labels ((set-loop-vars () (when (and line (< i llen)) (setf c (schar line i)) (let ((new-idx (+ 1 nl-match))) (declare (type fixnum new-idx)) (if (char= (etypecase *read-newline* (string (schar *read-newline* new-idx)) (character *read-newline*)) c) (setf nl-match new-idx) (setf nl-match -1)) (setf newline-matched? (= nl-len-1 nl-match))))) (current-last-char () (elt current (- (fill-pointer current) 1))) (store-char (char) (typecase char (character (vector-push-extend char current)) (string (iter (for c in-string char) (vector-push-extend c current))))) (finish-item () ;; trim off unquoted whitespace at the end (when (and (eql state :collecting) *trim-outer-whitespace*) (iter (while (white-space? (current-last-char))) (decf (fill-pointer current)))) ;; collect the result (let ((v (cons (if (and ;; got a zero length string? (zerop (length (string current))) ;; should we collect nil for zero length strings? (or (and (member state '(:waiting)) *unquoted-empty-string-is-nil*) (and (member state '(:waiting-for-next)) *quoted-empty-string-is-nil*))) (csv-data-read nil) (csv-data-read (copy-seq (string current)))) nil))) (if items (setf (cdr items-tail) v items-tail v) (setf items v items-tail v))) ;; go back to waiting for items (setf state :waiting) (setf (fill-pointer current) 0)) (skip-escape () (dotimes (j (- elen 1)) (next-char))) (read-line-in () (handler-bind ((end-of-file (lambda (sig) (ecase state (:waiting ;; let the signal go through, we have not read anything and already EOF (when items (finish-item) (return-from results items)) ) (:waiting-for-next ;; finished reading before encountering the next separator (return-from results items)) (:collecting ;; finished reading the file so must have finished this item (finish-item) (return-from results items)) (:collecting-quoted (restart-case (csv-parse-error "End of file while collecting quoted item: ~A" sig) (finish-item () (finish-item) (return-from results items)))))) )) ;; reset index, line and len for the next line of data (setf i 0) ;; we will increment immediately after this (if use-read-line? (multiple-value-bind (line-in didnt-get-a-newline?) (read-line in-stream) (setf line line-in read-line-got-a-newline? (not didnt-get-a-newline?) llen (length line))) (setf llen (read-into-buffer-until line in-stream :nl *read-newline* :nl-match nl-match))) (set-loop-vars))) (read-line-if-needed () (cond ;; if we dont have a line yet read one ((minusp llen) (read-line-in)) ;; we made it to the end of our buffer, so start again ((>= i llen) (ecase state (:collecting-quoted (when use-read-line? (when read-line-got-a-newline? (store-char *read-newline*)))) ((:waiting :collecting :waiting-for-next) (when newline-matched? (decf (fill-pointer current) nl-len)) (when (or newline-matched? use-read-line?) (finish-item) (return-from results items)))) (read-line-in)))) (next-char () (incf i) (read-line-if-needed) (unless (zerop i) (set-loop-vars))) (handle-character () (cond ;; read an empty line, next iteration ((and (member state '(:collecting :collecting-quoted)) (= i llen 0)) nil) ;; the next characters are an escape sequence, start skipping ((and (or (member state '(:collecting :collecting-quoted))) *quote-escape* ;; if this is null there is no escape (%escape-seq? line i *quote-escape* llen elen)) ;; this skips to the last char so that our next loop ;; will start at the next char (skip-escape) ;; TODO: *escape-mode* needs to happen on writing too (ecase *escape-mode* (:quote (store-char *quote*)) (:following ;; we need to immediately go to the next char and store ;; without processing (next-char) (store-char c)))) ;; the character is data separator, so gather the word unless ;; it is quoted data ((char= *separator* c) (ecase state (:collecting-quoted (store-char c)) ((:collecting :waiting :waiting-for-next) (finish-item)))) ;; the character is a quote (and not an escape) so start an item ;; finishing the item is the responsibility of separator/eol ((and *quote* (char= *quote* c)) (ecase state (:waiting (setf state :collecting-quoted)) (:collecting-quoted ;; if we end up trying to read (setf state :waiting-for-next)) (:collecting (csv-parse-error "we are reading non quoted csv data and found a quote at ~D~%~A" i line)))) (t ;; regular character (ecase state (:waiting (unless (and *trim-outer-whitespace* (white-space? c)) (setf state :collecting) (store-char c))) (:waiting-for-next (unless (and *trim-outer-whitespace* (white-space? c)) (csv-parse-error "We finished reading a quoted value and got more characters before a separator or EOL ~D~%~A" i line))) ((:collecting :collecting-quoted) (store-char c))))))) (next-char) (handle-character))))))) (iterate:defmacro-clause (for var in-csv input &optional skipping-header skip-first-p separator separator quote quote escaped-quote escaped-quote) "in-csv driver for iterate" (alexandria:with-unique-names (stream opened? skip) `(progn (with ,skip = ,skip-first-p) ;; can't bind values in a `with`, so listify and destructure (with (,stream ,opened?) = (multiple-value-list (%in-stream ,input))) (with *separator* = (or ,separator *separator*)) (with *quote* = (or ,quote *quote*)) (with *quote-escape* = (or ,escaped-quote *quote-escape*)) (finally-protected (when (and ,stream ,opened?) (close ,stream))) (handler-case (progn ;; optionally skip the first row (when (and ,skip (first-iteration-p)) (read-csv-row ,stream)) (for ,var = (restart-case (read-csv-row ,stream) (continue () :report "skip reading this row and try again on the next" (next-iteration)) (filter (new-row) :report "supply a different row to use instead of this erroring csv-row" new-row)))) (end-of-file () (finish)))))) (iterate:defmacro-clause (sampling expr &optional into var size size) "resevoir sample the input" (let ((sample (or var iterate::*result-var*))) (alexandria:with-unique-names (i sample-size sigil buffer row) `(progn (with ,sample) (with ,sample-size = (or ,size 100)) (with ,buffer = (make-array ,sample-size :initial-element ',sigil)) (with ,i = 0) (if (< ,i ,sample-size) (setf (aref ,buffer ,i) ,expr) (let ((r (random ,i))) (when (< r ,sample-size) (setf (aref ,buffer r) ,expr)))) (incf ,i) (finally ;; convert our sample to a list, but only if we actually took the sample (when (plusp ,i) (setf ,sample (iter (for ,row in-vector ,buffer) (until (eq ,row ',sigil)) (collect ,row))))))))) (defun read-csv-sample (stream-or-string sample-size &key row-fn map-fn skip-first-p ((:separator *separator*) *separator*) ((:quote *quote*) *quote*) ((:escape *quote-escape*) *quote-escape*) ((:unquoted-empty-string-is-nil *unquoted-empty-string-is-nil*) *unquoted-empty-string-is-nil*) ((:quoted-empty-string-is-nil *quoted-empty-string-is-nil*) *quoted-empty-string-is-nil*) ((:trim-outer-whitespace *trim-outer-whitespace*) *trim-outer-whitespace*) ((:newline *read-newline*) *read-newline*)) (iter (for row in-csv stream-or-string skipping-header skip-first-p) (sampling row into sample size sample-size) (finally (return (iter (for row in sample) (when map-fn (setf row (funcall map-fn row))) (when row-fn (funcall row-fn sample)) (collect row)))))) (defun read-csv (stream-or-string &key row-fn map-fn sample skip-first-p ((:separator *separator*) *separator*) ((:quote *quote*) *quote*) ((:escape *quote-escape*) *quote-escape*) ((:unquoted-empty-string-is-nil *unquoted-empty-string-is-nil*) *unquoted-empty-string-is-nil*) ((:quoted-empty-string-is-nil *quoted-empty-string-is-nil*) *quoted-empty-string-is-nil*) ((:trim-outer-whitespace *trim-outer-whitespace*) *trim-outer-whitespace*) ((:newline *read-newline*) *read-newline*) ((:escape-mode *escape-mode*) *escape-mode*)) "Read in a CSV by data-row (which due to quoted newlines may be more than one line from the stream) row-fn: passing this parameter will cause this read to be streaming and results will be discarded after the row-fn is called with data map-fn: used for manipulating the data by row during collection if specified; (funcall map-fn data) is collected instead of data sample: when a positive integer, only take that many samples from the input file skip-first-p: when true, skips the first line in the csv Keywords: separator: character separating between data cells. Defaults to *separator* quote: quoting character for text strings. Defaults to *quote* escape: escape character. Defaults to *quote-escape*" (if sample (read-csv-sample stream-or-string sample :row-fn row-fn :map-fn map-fn :skip-first-p skip-first-p) (iter (for data in-csv stream-or-string skipping-header skip-first-p) (if row-fn (funcall row-fn data) (collect (if map-fn (funcall map-fn data) data)))))) (defmacro do-csv ((row-var stream-or-pathname &rest read-csv-keys) &body body) "row-var: a variable that is passed into _body_ stream-or-pathname: a stream or a pathname to read the CSV data from read-csv-keys: keys and values passed to the _read-csv_ function body: body of the macro" `(read-csv ,stream-or-pathname ,@read-csv-keys :row-fn #'(lambda (,row-var) ,@body) ) ) ;; Copyright (c) 2011 Russ Tyndall , Acceleration.net http://www.acceleration.net ;; Copyright (c) 2002-2006, Edward Marco Baringer ;; All rights reserved. ;; ;; Redistribution and use in source and binary forms, with or without ;; modification, are permitted provided that the following conditions are ;; met: ;; ;; - Redistributions of source code must retain the above copyright ;; notice, this list of conditions and the following disclaimer. ;; ;; - Redistributions in binary form must reproduce the above copyright ;; notice, this list of conditions and the following disclaimer in the ;; documentation and/or other materials provided with the distribution. ;; ;; - Neither the name of Edward Marco Baringer, nor BESE, nor the names ;; of its contributors may be used to endorse or promote products ;; derived from this software without specific prior written permission. ;; ;; THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS ;; "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT ;; LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR ;; A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT ;; OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, ;; SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT ;; LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, ;; DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY ;; THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT ;; (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE ;; OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. cl-csv-20170403-git/data-table.lisp000066400000000000000000000031601306626277100166250ustar00rootroot00000000000000(cl:in-package :cl-csv) (defun get-data-table-from-csv (file &optional (has-column-names t) (munge-types t) sample &aux (dt (make-instance 'data-table:data-table))) "Gets a data-table object representing the CSV" (iter (for row in-csv file) (for data = (mapcar #'data-table::trim-and-nullify row)) (if (and has-column-names (first-iteration-p)) (setf (data-table:column-names dt) data) (if sample (sampling data into samples size sample) (collect data into rows))) (finally (setf (data-table:rows dt) (if sample samples rows)))) (if munge-types (data-table:coerce-data-table-of-strings-to-types dt) (data-table::ensure-column-data-types dt)) dt) (defun data-table-to-csv (dt &optional stream) "Write a datatable object out to csv" (write-csv (list* (data-table:column-names dt) (data-table:rows dt)) :stream stream)) (defun get-data-table-from-csv-list (list &optional (has-column-names t) (munge-types t) sample &aux (dt (make-instance 'data-table:data-table))) "Create a data-table from the parsed csv as lisp lists" (flet ((map-fn (row) (mapcar #'data-table::trim-and-nullify row))) (when has-column-names (setf (data-table:column-names dt) (map-fn (first list)))) (setf (data-table:rows dt) (mapcar (lambda (x) (if sample (map-fn (subseq x 0 sample)) (map-fn x))) (rest list))) (when munge-types (data-table:coerce-data-table-of-strings-to-types dt)) dt)) cl-csv-20170403-git/packages.lisp000066400000000000000000000011351306626277100164050ustar00rootroot00000000000000(cl:defpackage :cl-csv (:use :cl :cl-user :iterate) (:export :read-csv :csv-parse-error :format-csv-value :write-csv-value :write-csv-row :read-csv-row :write-csv :read-csv :*quote* :*separator* :*newline* :*quote-escape* :*empty-string-is-nil* #:read-csv-sample #:sampling #:data #:row ;; signals #:*enable-signals* #:filter #:csv-data-read #:csv-row-read ;; clsql stuff :export-query :import-from-csv :serial-import-from-csv ;; data table #:get-data-table-from-csv #:get-data-table-from-csv-list #:data-table-to-csv #:do-csv #:*default-external-format*))cl-csv-20170403-git/read-until.lisp000066400000000000000000000033341306626277100166760ustar00rootroot00000000000000(in-package :cl-csv) (defun read-into-buffer-until (buffer stream &key (nl #\newline) nl-match &aux (c #\null) (nl-idx (or nl-match -1)) (nl-len (etypecase nl (string (length nl)) (character 1))) (nl-len-1 (- nl-len 1)) (buffer-len (length buffer))) "This reads into a buffer until either the buffer is full or the we have read the newline character(s). If we read the newline characters they will be the last character(s) in the buffer " (declare (optimize (speed 3) (safety 0) (debug 0)) (type character c) (type (or simple-string character) nl) (type fixnum nl-len nl-len-1 nl-idx buffer-len) (type (simple-array character) buffer)) (dotimes (i buffer-len) (setf c (read-char stream nil *eof-char*)) ;; look for newlines (let ((new-idx (+ 1 nl-idx))) (declare (type fixnum new-idx)) (if (char= (etypecase nl (string (schar nl new-idx)) (character nl)) c) (setf nl-idx new-idx) (setf nl-idx -1))) (when (char= *eof-char* c) (if (zerop i) (error 'end-of-file :stream stream) ;; read some, then got an end of file (return-from read-into-buffer-until i))) (setf (schar buffer i) c) ;; got the nl (when (= nl-len-1 nl-idx) (return-from read-into-buffer-until (+ 1 i)))) ;; got a full buffer (return-from read-into-buffer-until buffer-len))cl-csv-20170403-git/tests/000077500000000000000000000000001306626277100151005ustar00rootroot00000000000000cl-csv-20170403-git/tests/bug18.csv000066400000000000000000000007311306626277100165440ustar00rootroot000000000000002014-11-07 10:02:17.302 CST,"gdt_new","gdt_insight_new",18445,"10.136.165.93:39143",545c1a19.480d,1,"COPY",2014-11-07 09:02:17 CST,6/1303255,1197337058,ERROR,57014,"canceling statement due to statement timeout",,,,,,"COPY (select t1.fspace, t3.fname, fad_target_classid, t2.name, num_client, num_client_big_imp, coverage from abc order by t1.fspace, t1.fad_target_classid) TO STDOUT DELIMITER ',' NULL 'null' CSV QUOTE '""'",,"ProcessInterrupts, postgres.c:3314","psql" cl-csv-20170403-git/tests/csv.lisp000066400000000000000000000432741306626277100165760ustar00rootroot00000000000000(defpackage :cl-csv-test (:use :cl :cl-user :cl-csv :lisp-unit2 :iter)) (in-package :cl-csv-test) (cl-interpol:enable-interpol-syntax) (defun run-all-tests () (run-tests :package :cl-csv-test :name :cl-csv :run-contexts #'with-summary-context )) (defmacro assert-length (exp it &rest them) `(assert-eql ,exp (length ,it) ,@them)) (defparameter +test-csv-quoted-path+ (asdf:system-relative-pathname :cl-csv "tests/test-csv-quoted.csv")) (defparameter +test-csv-unquoted-path+ (asdf:system-relative-pathname :cl-csv "tests/test-csv-unquoted.csv")) (defparameter +test-csv-unquoted-no-trailing-path+ (asdf:system-relative-pathname :cl-csv "tests/test-csv-unquoted-no-trailing.csv")) (defparameter +test-multiline+ (asdf:system-relative-pathname :cl-csv "tests/test-multiline-data.csv")) (defparameter +test-backslash-escapes+ (asdf:system-relative-pathname :cl-csv "tests/test-backslash-escapes.csv")) (defparameter +test-files+ (list +test-csv-quoted-path+ +test-csv-unquoted-path+ +test-csv-unquoted-no-trailing-path+)) (defparameter *test-csv1-rows* '(("first name" "last name" "job \"title\"" "number of hours" "id") ("Russ" "Tyndall" "Software Developer's, \"Position\"" "26.2" "1") ("Adam" "Smith" "Economist" "37.5" "2") ("John" "Doe" "Anonymous Human" "42.1" "3") ("Chuck" "Darwin" "Natural Philosipher" "17.68" "4") ("Bill" "Shakespear" "Bard" "12.2" "5") ("James" "Kirk" "Starship Captain" "13.1" "6") ("Bob" "Anon" "" "13.1" "6") ("Mr" "Iñtërnâtiônàlizætiøn" "" "1.1" "0"))) (defparameter *test-csv1* "\"first name\",\"last name\",\"job \"\"title\"\"\",\"number of hours\",\"id\" \"Russ\",\"Tyndall\",\"Software Developer's, \"\"Position\"\"\",\"26.2\",\"1\" \"Adam\",\"Smith\",\"Economist\",\"37.5\",\"2\" \"John\",\"Doe\",\"Anonymous Human\",\"42.1\",\"3\" \"Chuck\",\"Darwin\",\"Natural Philosipher\",\"17.68\",\"4\" \"Bill\",\"Shakespear\",\"Bard\",\"12.2\",\"5\" \"James\",\"Kirk\",\"Starship Captain\",\"13.1\",\"6\" \"Bob\",\"Anon\",\"\",\"13.1\",\"6\" \"Mr\",\"Iñtërnâtiônàlizætiøn\",\"\",\"1.1\",\"0\" ") (defparameter *test-csv1-v2* "first name,last name,\"job \"\"title\"\"\",number of hours,id Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1 Adam,Smith,Economist,37.5,2 John,Doe,Anonymous Human,42.1,3 Chuck,Darwin,Natural Philosipher,17.68,4 Bill,Shakespear,Bard,12.2,5 James,Kirk,Starship Captain,13.1,6 Bob,Anon,,13.1,6 Mr,Iñtërnâtiônàlizætiøn,,1.1,0 ") (defparameter *test-csv-no-trailing-newline* "first name,last name,\"job \"\"title\"\"\",number of hours,id Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1") (defparameter *test-csv-data-with-newlines* "first name,last name,\"job \"\"title\"\"\",number of hours,id Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1") (defparameter *test-csv-data-waiting-next-error* "\"Which of the following is an appropriate calming technique or statement: A. \"\"I can help you.\"\" B. \"\"Shut up.\"\" C. \"\"If you don't calm down I'm not sending anyone.\"\" D. \"\"Ma'am, ma'am\ ma'am!\"\"\",A") (define-test parsing-1 (:tags '(parsing)) (assert-equal *test-csv1-rows* (read-csv *test-csv1*)) (assert-equal *test-csv1-rows* (read-csv *test-csv1-v2*))) (define-test writing-1 (:tags '(writing)) (assert-equal *test-csv1* (write-csv *test-csv1-rows* :always-quote t))) (define-test parsing-errors (:tags '(parsing errors)) (assert-error 'csv-parse-error (read-csv-row "first name, a test\" broken quote, other stuff")) (assert-error 'csv-parse-error (read-csv-row "first name,\"a test broken quote\" what are these chars, other stuff")) (assert-error 'csv-parse-error (read-csv-row "first name,\"a test unfinished quote, other stuff")) (assert-eql 3 (length (read-csv-row "first name, \"a test broken quote\", other stuff "))) ) (define-test no-trailing-parse (:tags '(parsing errors)) (let* ((data (read-csv *test-csv-no-trailing-newline*)) (str (write-csv data :always-quote t)) (data2 (read-csv str))) (assert-equal 2 (length data)) (assert-equal 5 (length (first data))) (assert-equal 5 (length (second data))) (assert-equal data data2))) (define-test data-with-newlines (:tags '(whitespace parsing writing)) (let* ((data (read-csv *test-csv-data-with-newlines*)) (str (write-csv data :always-quote t)) (data2 (read-csv str))) (assert-equal 2 (length data)) (assert-equal 5 (length (first data))) (assert-equal 5 (length (second data))) (assert-equal "Software Developer's, \"Position\"" (third (second data))) (assert-equal data data2))) (define-test data-with-whitespace-trim (:tags '(whitespace parsing trim)) (assert-equal '("first" "last" " other " "" nil nil) (read-csv-row " first , last , ' other ','',, " :unquoted-empty-string-is-nil t :quoted-empty-string-is-nil nil :trim-outer-whitespace t :quote #\')) (assert-equal '(" first " " last " " other " "" nil " ") (read-csv-row " first , last ,' other ','',, " :unquoted-empty-string-is-nil t :quoted-empty-string-is-nil nil :trim-outer-whitespace nil :quote #\')) (assert-error 'csv-parse-error (read-csv-row " first , last , ' other ','',, " :unquoted-empty-string-is-nil t :quoted-empty-string-is-nil nil :trim-outer-whitespace nil :quote #\') "whitespace before quoted values is a parse error if we are not trimming ") (assert-error 'csv-parse-error (read-csv-row " first , last ,' other ' ,'',, " :unquoted-empty-string-is-nil t :quoted-empty-string-is-nil nil :trim-outer-whitespace nil :quote #\') "whitespace after quoted values is a parse error if we are not trimming ") ) (define-test data-with-whitespace-nilling (:tags '(whitespace parsing trim)) (assert-equal '("first" "last" " other " nil nil nil) (read-csv-row " first , last , ' other ' ,'',, " :quoted-empty-string-is-nil t :unquoted-empty-string-is-nil t :quote #\')) (assert-equal '("first" "last" " other " "" "" "") (read-csv-row " first , last ,' other ','',, " :quoted-empty-string-is-nil nil :unquoted-empty-string-is-nil nil :quote #\')) (assert-equal '("first" "last" " other " nil "" "") (read-csv-row " first , last , ' other ','',, " :quoted-empty-string-is-nil T :unquoted-empty-string-is-nil nil :quote #\') "whitespace before quoted values is a parse error if we are not trimming ") (assert-equal '("first" "last" " other " "" nil nil) (read-csv-row " first , last ,' other ' ,'',, " :quoted-empty-string-is-nil nil :unquoted-empty-string-is-nil t :quote #\') "whitespace after quoted values is a parse error if we are not trimming ") ) (define-test files (:tags '(parsing files)) (iter (for csv in +test-files+) (for data = (read-csv csv)) (assert-equal *test-csv1-rows* data csv))) (define-test multi-line-file (:tags '(parsing files)) (let ((data (read-csv +test-multiline+))) (assert-equal 2 (length data) data) (assert-equal "test of multiline" (nth 3 (first data)) )) ) (define-test dont-always-quote-and-newline (:tags '(writing whitespace quotation)) (let* ((row '("Russ" "Tyndall" "Software Developer's, \"Position\"" "26.2" "1" ",")) (res (write-csv-row row :always-quote nil :newline #?"\n"))) (assert-equal #?"Russ,Tyndall,\"Software Developer's, \"\"Position\"\"\",26.2,1,\",\"\n" res))) (define-test dont-always-quote-and-newline-2 (:tags '(writing whitespace quotation)) (let* ((row '("," #?"a\r\nnewline\r\ntest\r\n")) (res (write-csv-row row :always-quote nil :newline #?"\n"))) (assert-equal #?"\",\",\"a\r\nnewline\r\ntest\r\n\"\n" res))) (define-test cause-error (:tags '(parsing errors)) (let ((data (read-csv *test-csv-data-waiting-next-error*))) (assert-true data))) (define-test chars-in-test (:tags '(utils parsing)) (assert-true (cl-csv::chars-in "a" "abcdef")) (assert-false (cl-csv::chars-in "qu" "abcdef")) (assert-true (cl-csv::chars-in "qu" "asdfqasdf")) (assert-true (cl-csv::chars-in "qu" "asdfuasdf")) (assert-true (cl-csv::chars-in (list "q" "u") "asdfuasdf")) (assert-true (cl-csv::chars-in (list #\q #\u) "asdfuasdf")) (assert-true (cl-csv::chars-in (list "q" #\u) "asdfqasdf"))) (define-test iterate-clauses (:tags '(utils iterate)) (iter (for (a b c) in-csv "1,2,3 4,5,6") (assert-equal (if (first-time-p) "1" "4") a) (assert-equal (if (first-time-p) "2" "5") b) (assert-equal (if (first-time-p) "3" "6") c) (for i from 0) (finally (assert-equal 1 i))) ;; test SKIPPING-HEADER option (iter (for (a b c) in-csv "1,2,3 4,5,6" SKIPPING-HEADER T) (assert-equal "4" a) (assert-equal "5" b) (assert-equal "6" c) (for i from 0) (finally (assert-equal 0 i))) ;; test SEPARATOR (iter (for (a b c) in-csv "1|2|3 4|5|6" SKIPPING-HEADER T SEPARATOR #\|) (assert-equal "4" a) (assert-equal "5" b) (assert-equal "6" c) (for i from 0) (finally (assert-equal 0 i)))) (define-test sampling-iterate (:tags '(parsing iterate)) (assert-length 9 (iter (for row in-csv *test-csv1*) (cl-csv:sampling row))) (assert-length 2 (iter (for row in-csv *test-csv1*) (cl-csv:sampling row into sample size 2) (finally (return sample)))) (assert-length 2 (read-csv-sample *test-csv1* 2)) (assert-length 3 (iter (for row in-csv *test-csv1* skipping-header t) (cl-csv::sampling row size 3))) (assert-length 9 (iter (for row in-csv *test-csv1*) (cl-csv:sampling row into sample size 25) (finally (return sample))))) (define-test csv-signal-enabling (:tags '(signals)) (assert-signal 'csv-row-read (assert-signal 'csv-data-read (let ((*enable-signals* t)) (cl-csv:read-csv "1,2,3")))) (assert-no-signal 'csv-row-read (assert-no-signal 'csv-data-read (let ((*enable-signals* nil)) (cl-csv:read-csv "1,2,3"))))) (define-test csv-filter (:tags '(signals)) (assert-equal '(1 2 3) (let ((*enable-signals* t)) (handler-bind ((csv-data-read (lambda (c) (invoke-restart 'filter (parse-integer (cl-csv::data c)))))) (cl-csv:read-csv-row "1,2,3")))) (assert-equal '(1 2 3) (let ((*enable-signals* t)) (handler-bind ((csv-row-read (lambda (c) (invoke-restart 'filter (mapcar #'parse-integer (cl-csv::row c)))))) (cl-csv:read-csv-row "1,2,3"))))) (defun displaced-sub-string (s &key (start 0) (end (length s))) (make-array (- end start) :element-type (array-element-type s) :displaced-to s :displaced-index-offset start)) (define-test csv-continue-signals (:tags '(signals)) (handler-bind ((csv-parse-error #'continue)) (assert-equal '(("1" "2" "3") ("3" "4" "5")) (cl-csv:read-csv "1,2,3 2,3',4 3,4,5" :quote #\')))) (define-test early-end-of-stream (:tags '(errors parsing)) (let ((line #?|"1","2|)) (assert-error 'cl-csv:csv-parse-error (cl-csv:read-csv-row line))) (let ((line "")) (assert-error 'end-of-file (cl-csv:read-csv-row line))) (let ((line #?|"1","2|)) (assert-equal '("1" "2") (handler-bind ((cl-csv:csv-parse-error (lambda (c) (declare (ignore c)) (invoke-restart 'cl-csv::finish-item)))) (cl-csv:read-csv-row line)))) ) (define-test read-into-buffer-until-test (:tags '(read-until)) ;; \r\l newline (with-input-from-string (in #?"test this\r\n thing") (let* ((s (make-string 80)) (l (cl-csv::read-into-buffer-until s in :nl #?"\r\n"))) (assert-eql 11 l) (assert-equal #?"test this\r\n" (displaced-sub-string s :end l)))) ;; newline (with-input-from-string (in #?"t\nest this\n thing") (let* ((s (make-string 80)) (l (cl-csv::read-into-buffer-until s in :nl #\newline))) (assert-eql 2 l) (assert-equal #?"t\n" (displaced-sub-string s :end l)) (let ((l (cl-csv::read-into-buffer-until s in :nl #\newline))) (assert-eql 9 l) (assert-equal #?"est this\n" (displaced-sub-string s :end l))))) ;; EOF (with-input-from-string (in #?"test this thing") (let* ((s (make-string 80)) (l (cl-csv::read-into-buffer-until s in :nl #\newline))) (assert-eql (length "test this thing") l) (assert-equal "test this thing" (displaced-sub-string s :end l)) )) ;; filled buffer (with-input-from-string (in #?"test this thing") (let* ((s (make-string 4)) (l (cl-csv::read-into-buffer-until s in :nl #\newline))) (assert-eql 4 l) (assert-equal "test" (displaced-sub-string s :end l)) (assert-eql 4 (cl-csv::read-into-buffer-until s in :nl #\newline)) (assert-eql 4 (cl-csv::read-into-buffer-until s in :nl #\newline)) (assert-eql 3 (cl-csv::read-into-buffer-until s in :nl #\newline)) (assert-error 'end-of-file (cl-csv::read-into-buffer-until s in :nl #\newline)) ))) (define-test buffer-spanning-new-lines (:tags '(read-until whitespace parsing)) (with-input-from-string (in "testRNtest") (let* ((s (make-string 5)) len) (setf len (cl-csv::read-into-buffer-until s in :nl "RN")) (assert-eql 5 len) (setf len (cl-csv::read-into-buffer-until s in :nl "RN" :nl-match 0)) (assert-eql 1 len ) (setf len (cl-csv::read-into-buffer-until s in :nl "RN")) (assert-eql 4 len)))) (define-test buffer-spanning-new-lines2 (:tags '(read-until newlines whitespace parsing)) ;; ** newline (with-input-from-string (in "test**tes**te**test") (let* ((s (make-string 5)) len (nl-idx -1)) (flet ((rebind ( &optional new-nl-idx) (when new-nl-idx (setf nl-idx new-nl-idx)) (multiple-value-setq (len) (cl-csv::read-into-buffer-until s in :nl "**" :nl-match nl-idx)))) (rebind) (assert-eql 5 len :first s) (rebind 0) (assert-eql 1 len :second) (rebind -1) (assert-eql 5 len :third) (rebind) (assert-eql 4 len :third)) ))) (define-test different-newlines (:tags '(read-until newlines whitespace parsing)) (with-input-from-string (in "a|b|c|d**1|2|3|4") (let* ((cl-csv::*buffer-size* 8) (rows (cl-csv:read-csv in :newline "**" :separator "|"))) (assert-equal 2 (length rows)) (assert-equal '("a" "b" "c" "d") (first rows)) (assert-equal '("1" "2" "3" "4") (second rows))) ) (with-input-from-string (in "a|b|c|d**1|2|3|4") (let ((rows (cl-csv:read-csv in :newline "**" :separator "|"))) (assert-equal 2 (length rows)) (assert-equal '("a" "b" "c" "d") (first rows)) (assert-equal '("1" "2" "3" "4") (second rows))) ) (with-input-from-string (in "a|b|c|d*1|2|3|4") (let ((rows (cl-csv:read-csv in :newline #\* :separator #\|))) (assert-equal 2 (length rows)) (assert-equal '("a" "b" "c" "d") (first rows)) (assert-equal '("1" "2" "3" "4") (second rows))) ) (with-input-from-string (in "a|b|c|d*1|2|3|4") (let ((rows (cl-csv:read-csv in :newline "*" :separator "|"))) (assert-equal 2 (length rows)) (assert-equal '("a" "b" "c" "d") (first rows)) (assert-equal '("1" "2" "3" "4") (second rows))) )) (define-test backslash-escapes (:tags '(backslash escapes parsing)) (lisp-unit2:assert-error 'csv-parse-error (cl-csv:read-csv +test-backslash-escapes+ :escape #?|\"|)) (let ((results (cl-csv:read-csv +test-backslash-escapes+ :escape #\\ :escape-mode :following ))) (assert-equal 2 (length results)) (assert-equal `("id","timestamp","date","comment","something") (first results)) )) (defparameter +test-csv-bug18-path+ (asdf:system-relative-pathname :cl-csv "tests/bug18.csv")) (define-test issue-18 (:tags '(parsing bugs whitespace empty-line)) (let* ((results (cl-csv:read-csv-row +test-csv-bug18-path+ :separator #\, :escape "\"\"")) (long-ml (nth 19 results))) (assert-true results) (assert-equal "COPY (select t1.fspace, t3.fname, fad_target_classid, t2.name, num_client, num_client_big_imp, coverage from abc order by t1.fspace, t1.fad_target_classid) TO STDOUT DELIMITER ',' NULL 'null' CSV QUOTE '\"'" long-ml) (assert-equal '("2014-11-07 10:02:17.302 CST" "gdt_new" "gdt_insight_new" "18445" "10.136.165.93:39143" "545c1a19.480d" "1" "COPY" "2014-11-07 09:02:17 CST" "6/1303255" "1197337058" "ERROR" "57014" "canceling statement due to statement timeout" "" "" "" "" "" "COPY (select t1.fspace, t3.fname, fad_target_classid, t2.name, num_client, num_client_big_imp, coverage from abc order by t1.fspace, t1.fad_target_classid) TO STDOUT DELIMITER ',' NULL 'null' CSV QUOTE '\"'" "" "ProcessInterrupts, postgres.c:3314" "psql") results))) (defparameter +test-tab-csv-issue-10+ "id inches name 1 72\" Russ Tyndall 2 67\" Amy Bobanolis ") (define-test issue-10-tab-csv (:tags '(parsing bugs whitespace empty-line)) (let* ((csv (read-csv +test-tab-csv-issue-10+ :quote #\nul :separator #\tab)) (row1 (second csv))) (assert-equal "72\"" (second row1) csv))) cl-csv-20170403-git/tests/line-endings.csv000066400000000000000000000000121306626277100201620ustar00rootroot00000000000000test testcl-csv-20170403-git/tests/speed.lisp000066400000000000000000000121271306626277100170740ustar00rootroot00000000000000(defpackage :cl-csv-test.speed-tests (:use :cl :cl-user :cl-csv :lisp-unit2 :iter)) (in-package :cl-csv-test.speed-tests) (defun run-speed-tests () (lisp-unit2:run-tests :package :cl-csv-test.speed-tests :name :cl-csv-speed-tests :run-contexts #'lisp-unit2:with-summary-context)) (defun log-time (&optional (time (get-universal-time)) stream) "returns a date as ${mon}/${d}/${y} ${h}:${min}:{s}, defaults to get-universal-time" (multiple-value-bind ( s min h ) (decode-universal-time time) (format stream "~2,'0d:~2,'0d:~2,'0d " h min s))) (defun test-log (message &rest args) (format *standard-output* "~&") (log-time (get-universal-time) *standard-output*) (apply #'format *standard-output* message args) (format *standard-output* "~%")) (eval-when (:compile-toplevel :load-toplevel :execute) (defmacro log-around ((log-name message &rest args) &body body) "Logs the beginning and end of a body. ARGS are evaluated twice" (let ((gmessage (gensym "GMESSAGE-"))) `(let ((,gmessage ,message)) (flet ((msg (&optional tag) (format nil "~A ~a" tag ,gmessage))) (,log-name (msg "BEGIN") ,@args) (multiple-value-prog1 (progn ,@body) (,log-name (msg " END") ,@args)))))) (defmacro time-and-log-around ((log-name message &rest args) &body body) "Logs the beginning and end of a body. ARGS are evaluated twice" (let ((trace-output (gensym "TRACE-OUTPUT-"))) `(let (,trace-output) ;;leave nil so the first log call doesn't print an extra newline (log-around (,log-name ,(concatenate 'string message "~@[~%~a~]") ,@args ,trace-output) (setf ,trace-output (make-array 10 :element-type 'character :adjustable T :fill-pointer 0)) (with-output-to-string (*trace-output* ,trace-output) (time (progn ,@body)))))))) (defparameter +test-big-file+ (asdf:system-relative-pathname :cl-csv "tests/long-test.csv")) (define-test write-big-file () (let ((n 120000)) (time-and-log-around (test-log "write large file test") (with-open-file (s +test-big-file+ :direction :output :if-exists :supersede ) (iter (for i from 0 to n) (write-csv-row '("Russ" "Tyndall" "Software Developer's, \"Position\"" "26.2" "1" "further columns" "even" "more" "data") :stream s)))))) (define-test count-big-file-csv-rows () (let ((cnt 0)) (time-and-log-around (test-log "read large file test") (read-csv +test-big-file+ :row-fn (lambda (r) (declare (ignore r)) (incf cnt)) )) cnt)) (define-test read-by-line-and-buffer (:tags '(cl-csv-test::read-until)) (let ((cnt 0) (cnt2 0) (cnt3 0)) (time-and-log-around (test-log "read large file by lines") (let ( line) (cl-csv::with-csv-input-stream (s +test-big-file+ ) (handler-case (loop while (setf line (read-line s)) do (incf cnt)) (end-of-file (c) (declare (ignore c))))))) (time-and-log-around (test-log "read large file by buffer") (cl-csv::with-csv-input-stream (s +test-big-file+ ) (iter (with buffer = (make-array 80 :element-type 'character )) (with fill) (handler-case (setf fill (read-sequence buffer s)) (end-of-file () (finish))) (incf cnt2) (while (= 80 fill)) ))) (time-and-log-around (test-log "read large file by read-into-buffer-until") (cl-csv::with-csv-input-stream (s +test-big-file+ ) (let ((buffer (make-string cl-csv::*buffer-size*))) (handler-case (loop while (plusp (cl-csv::read-into-buffer-until buffer s)) do (incf cnt3)) (end-of-file (c) (declare (ignore c))))))) (format lisp-unit2:*test-stream* "~@:_ lines:~D , buffers:~D, buffered-lines:~D~@:_" cnt cnt2 cnt3))) (define-test collect-big-file-csv-rows () (time-and-log-around (test-log "read large file test") (read-csv +test-big-file+)) nil ; so we dont print 10m to the repl ) (defun process-csv2 (csv-name process-function ) (let ( collector ) (labels ((row-processor (row) (let ((result (funcall process-function row))) (when result (push result collector))))) (cl-csv:read-csv csv-name :row-fn #'row-processor) collector))) (defun return-blanks2 (filename) (labels ((row-checker (row &aux (6th (sixth row)) ) (if (string-equal 6th "") (list (first row) (sixth row))))) (process-csv2 filename #'row-checker))) (defun test-pnathan-code2 () (time-and-log-around (test-log "test-pnathan2 read") (return-blanks2 +test-big-file+))) ;; (test-pnathan-code) ;; 15:57:18 BEGIN test-pnathan read ;; 15:57:21 END test-pnathan read ;; Evaluation took: ;; 3.480 seconds of real time ;; 3.390000 seconds of total run time (3.200000 user, 0.190000 system) ;; [ Run times consist of 0.170 seconds GC time, and 3.220 seconds non-GC time. ] ;; 97.41% CPU ;; 8,678,145,157 processor cycles ;; 229,327,728 bytes consed cl-csv-20170403-git/tests/test-backslash-escapes.csv000066400000000000000000000002121306626277100221410ustar00rootroot00000000000000"\id","t\imestamp","da\te","comm\ent","somethin\g" "16417153","1401640227","\"Jun 1 2014\"","HTML -//W3C//DTD HTML 4.01 Frameset//EN\\",""cl-csv-20170403-git/tests/test-csv-quoted.csv000066400000000000000000000006531306626277100206700ustar00rootroot00000000000000"first name","last name","job ""title""","number of hours","id" "Russ","Tyndall","Software Developer's, ""Position""","26.2","1" "Adam","Smith","Economist","37.5","2" "John","Doe","Anonymous Human","42.1","3" "Chuck","Darwin","Natural Philosipher","17.68","4" "Bill","Shakespear","Bard","12.2","5" "James","Kirk","Starship Captain","13.1","6" "Bob","Anon","","13.1","6" "Mr","Iñtërnâtiônàlizætiøn","","1.1","0" cl-csv-20170403-git/tests/test-csv-unquoted-no-trailing.csv000066400000000000000000000005131306626277100234470ustar00rootroot00000000000000first name,last name,"job ""title""",number of hours,id Russ,Tyndall,"Software Developer's, ""Position""",26.2,1 Adam,Smith,Economist,37.5,2 John,Doe,Anonymous Human,42.1,3 Chuck,Darwin,Natural Philosipher,17.68,4 Bill,Shakespear,Bard,12.2,5 James,Kirk,Starship Captain,13.1,6 Bob,Anon,,13.1,6 Mr,Iñtërnâtiônàlizætiøn,,1.1,0cl-csv-20170403-git/tests/test-csv-unquoted.csv000066400000000000000000000005141306626277100212270ustar00rootroot00000000000000first name,last name,"job ""title""",number of hours,id Russ,Tyndall,"Software Developer's, ""Position""",26.2,1 Adam,Smith,Economist,37.5,2 John,Doe,Anonymous Human,42.1,3 Chuck,Darwin,Natural Philosipher,17.68,4 Bill,Shakespear,Bard,12.2,5 James,Kirk,Starship Captain,13.1,6 Bob,Anon,,13.1,6 Mr,Iñtërnâtiônàlizætiøn,,1.1,0 cl-csv-20170403-git/tests/test-multiline-data.csv000066400000000000000000000001111306626277100214740ustar00rootroot00000000000000this,is,a,"test of multiline", data row2,of,the,"test of multiline", datacl-csv-20170403-git/vars.lisp000066400000000000000000000034231306626277100156040ustar00rootroot00000000000000(in-package :cl-csv) (cl-interpol:enable-interpol-syntax) (defvar *quote* #\" "Default quote character") (defvar *separator* #\, "Default separator character") (defvar *escape-mode* :quote "Controls how escapes are handled. :quote - replace the entire *quote-escape* sequence with the quote character whenever we find it. Commonly used with \"\" quote escapes :following - replace the escape character and the following character with just the following character. EG: (*quote-escape* #\\ ) \\ -> \ \r -> r \' -> ' ") ;; we want to read basically anything by default (defvar *read-newline* #?"\n" "Default newline string for reading. We trim extra whitespace by default *trim-outer-whitespace*") ;; we want to write toward excel by default (defvar *write-newline* #?"\r\n" "When writing what should the newline convention be ") (defvar *always-quote* nil "Default setting for always quoting") (defvar *quote-escape* #?"${ *quote* }${ *quote* }" "Default setting for escaping quotes") (defvar *unquoted-empty-string-is-nil* nil "Should unquoted empty string values, be nil or \"\".") (defvar *quoted-empty-string-is-nil* nil "Should empty string values, be nil or \"\". Unquoted values are always trimmed of surrounding whitespace. Quoted values are never be trimmed") (defvar *trim-outer-whitespace* t "Should white space between delimiters and data or quotes be removed These underscores (if they were spaces) are the locations in question 'a',_b_,_' c '_,_d ") (defvar *enable-signals* nil "Should the reading and writing process enable filtering signals") (defvar *eof-char* #\null "The char we use for eof") (defvar *buffer-size* 512)