cl-markdown-20101006-darcs/ 0000755 0001750 0001750 00000000000 11453110206 013676 5 ustar xach xach cl-markdown-20101006-darcs/test-results/ 0000755 0001750 0001750 00000000000 11453110206 016354 5 ustar xach xach cl-markdown-20101006-darcs/dev/ 0000755 0001750 0001750 00000000000 11453110206 014454 5 ustar xach xach cl-markdown-20101006-darcs/dev/epilogue.lisp 0000644 0001750 0001750 00000000201 11453110206 017147 0 ustar xach xach ;;; the last file to be loaded...
(in-package #:cl-markdown)
(setf *parsing-environment* (make-instance 'parsing-environment))
cl-markdown-20101006-darcs/dev/multiple-documents.lisp 0000644 0001750 0001750 00000017347 11453110206 021213 0 ustar xach xach (in-package #:cl-markdown)
(defun markdown-many (pairs &rest args
&key format additional-extensions render-extensions
&allow-other-keys)
"Markdown-many processes several documents simultaneously as if it
was processing one large document. Its chief purpose is to make it easy to
create inter-document links. Markdown-many takes as input
* `pairs` - a list of lists where each sublist contains the markdown
file to be processed as `input` in its first element and the name of
the file to be produced as the `output`.
* `:format` - a keyword argument specifying the kind of output document
to produce
* `:additional-extensions` - a list of extensions that should be active
both while parsing and rendering.
* `:render-extensions` - a list of extensions that should be active
during rendering.
Here is an example: suppose document-1.md contains
# Document-1
See [document-2][] for details.
and document-2.md contains
# Document 2
[Document-1][] provides an overview.
Getting these links to work using only Markdown will require added explicit
reference link information that will be tied to the file _names_. Markdown-many,
on the other hand, will automatically combine the link information and
processes it automatically.
"
(let ((main-document (make-instance 'multi-document))
(docs nil))
(setf docs
(loop for datum in pairs collect
(bind (((source destination &rest doc-args) datum))
(format t "~&Parsing: ~s~%" source)
(list (apply #'markdown source
:document-class 'child-document
:parent main-document
:format :none (merge-arguments args doc-args))
destination))))
;; transfer information from docs to the parent
(loop for (doc destination) in docs do
(transfer-document-data main-document doc destination))
;; render 'em
(loop for (doc destination) in docs do
(format t "~&Rendering: ~s" destination)
(let ((*current-document* doc)
(*render-active-functions*
(mapcar #'canonize-command
(or render-extensions
(if additional-extensions
`(,@additional-extensions
,@*render-active-functions*)
*render-active-functions*)))))
(render-to-stream doc format destination)))
(setf (children main-document) (mapcar #'first docs))
(values main-document docs)))
(defun merge-arguments (args-1 args-2)
(let ((result args-1))
(map-window-over-elements
args-2 2 2
(lambda (pair)
(bind (((key value) pair)
(present (getf result key)))
(setf (getf result key)
(if present
(append (ensure-list present) (ensure-list value))
value)))))
result))
#+(or)
(merge-arguments '(:a (1) :b (2)) '(:c (3) :a (2)))
#+(or)
(defun _render-one (doc)
(let ((*current-document* doc)
(*render-active-functions*
(mapcar #'canonize-command
`(cl-markdown::docs cl-markdown::docs-index
cl-markdown::today cl-markdown::now
cl-markdown::glossary
,@*render-active-functions*))))
(render-to-stream doc :html #p"/tmp/one.html")))
#+(or)
(untrace markdown)
#+(or)
(compile 'markdown-many)
#+(or)
(cl-markdown:markdown-many
`((,(system-relative-pathname 'cl-markdown "dev/md1.md")
,(system-relative-pathname 'cl-markdown "dev/md1.html"))
(,(system-relative-pathname 'cl-markdown "dev/md2.md")
,(system-relative-pathname 'cl-markdown "dev/md2.html")))
:format :html)
(defun transfer-document-data (parent child destination)
(transfer-link-info parent child destination)
(transfer-selected-properties
parent child
(set-difference (collect-keys (properties child))
(list :footnote :style-sheet :style-sheets :title)))
(transfer-document-metadata parent child))
(defun transfer-document-metadata (parent child)
(iterate-key-value
(metadata child)
(lambda (key value)
; (print (list :p (item-at-1 (metadata parent) key)
; :c value))
(aif (item-at-1 (metadata parent) key)
(setf (item-at-1 (metadata parent) key) (merge-entries it value))
(setf (item-at-1 (metadata parent) key) value)))))
(defun transfer-selected-properties (parent child properties)
(let ((*current-document* parent))
(iterate-elements
properties
(lambda (property)
(when (item-at-1 (properties child) property)
(setf (document-property property)
(first (item-at-1 (properties child) property))))))))
(defun transfer-link-info (parent child destination)
(let ((*current-document* parent))
(iterate-key-value
(link-info child)
(lambda (id info)
(setf (item-at (link-info parent) id)
(transfer-1-link-info info parent child destination))))))
(defgeneric transfer-1-link-info (info parent child destination))
(defmethod transfer-1-link-info ((info link-info) parent child destination)
(declare (ignore parent child))
(make-instance 'link-info
:id (id info)
:url (if (relative-url-p (url info))
(format nil "~@[~a~]~@[.~a~]~a"
(pathname-name destination)
(pathname-type destination)
(url info))
(url info))
:title (title info)
:properties (properties info)))
(defun relative-url-wrt-destination (url destination)
(if (relative-url-p url)
(format nil "~@[~a~]~@[.~a~]~a"
(pathname-name destination)
(pathname-type destination)
url)
url))
(defun relative-url-p (url)
;; FIXME -- look at the spec...
(not
(or (starts-with url "http:")
(starts-with url "mailto:")
(starts-with url "file:"))))
(defmethod transfer-1-link-info ((info extended-link-info)
parent child destination)
(declare (ignore parent child destination))
(make-instance 'extended-link-info
:id (id info)
:kind (kind info)
:contents (contents info)))
;;;
;; A slightly horrid hack that is good enough for indices but
;; completely untested
(defgeneric ugly-create-from-template (thing)
)
(defmethod ugly-create-from-template ((thing standard-object))
(make-instance (class-of thing)))
(defgeneric merge-entries (a b)
(:documentation "Returns a new container C \(of the same type as `a`\)
such that C contains every *entry* in a and b. C may share structure with
`a` and `b`."))
(defmethod merge-entries :around ((a t) (b t))
; (print (list :me a b))
(call-next-method))
(defmethod merge-entries ((a null) (b t))
b)
(defmethod merge-entries ((a null) (b iteratable-container-mixin))
(error "not implemented"))
(defmethod merge-entries ((a null) (b key-value-iteratable-container-mixin))
(merge-using-key-value (ugly-create-from-template b) b))
(defmethod merge-entries ((a t) (b t))
(cond ((and (key-value-iteratable-p a)
(key-value-iteratable-p b))
#+(or)
(merge-key-value-via-iteration a b)
(error "not implemented"))
((and (iteratable-p a)
(iteratable-p b))
(merge-elements-via-iteration a b))
(t
;; FIXME - drop b?
a)))
(defmethod merge-entries ((a list) (b t))
(append a (list b)))
(defmethod merge-entries ((a list) (b list))
(merge-elements-via-iteration a b))
(defmethod merge-entries ((a iteratable-container-mixin)
(b iteratable-container-mixin))
(merge-elements-via-iteration a b))
(defmethod merge-entries
((a key-value-iteratable-container-mixin)
(b key-value-iteratable-container-mixin))
(let ((new (ugly-create-from-template a)))
(merge-using-key-value new a)
(merge-using-key-value new b)
new))
(defun merge-elements-via-iteration (a b)
(let ((new (ugly-create-from-template a)))
(iterate-elements a (lambda (elt) (insert-item new elt)))
(iterate-elements b (lambda (elt) (insert-item new elt)))
new))
(defun merge-using-key-value (a b)
(iterate-key-value b (lambda (key value)
(let ((existing (item-at a key)))
(setf (item-at a key)
(if existing
(merge-entries existing value)
value)))))
a)
cl-markdown-20101006-darcs/dev/spans.lisp 0000644 0001750 0001750 00000023415 11453110206 016476 0 ustar xach xach (in-package #:cl-markdown)
(defvar *current-span* nil)
(defstruct (markdown-scanner (:conc-name scanner-))
name regex priority function)
(setf (item-at-1 *spanner-parsing-environments* 'default)
(make-instance
'sorted-list-container
:sorter '<
:key 'scanner-priority
:initial-contents
`(,(make-markdown-scanner
:regex (create-scanner '(:sequence escaped-character))
:name 'escaped-character
:priority 1
:function 'convert-escape-temporarily)
,(make-markdown-scanner
:regex (create-scanner '(:sequence inline-image))
:name 'inline-image
:priority 2)
,(make-markdown-scanner :regex (create-scanner
'(:sequence reference-image))
:name 'reference-image
:priority 3)
,(make-markdown-scanner :regex (create-scanner
'(:sequence coded-reference-link))
:name 'code
:priority 4)
,(make-markdown-scanner :regex (create-scanner
'(:sequence inline-link))
:name 'inline-link
:priority 5)
,(make-markdown-scanner :regex (create-scanner
'(:sequence reference-link))
:name 'reference-link
:priority 6)
,(make-markdown-scanner :regex (create-scanner '(:sequence backtick))
:name 'code
:priority 7)
,(make-markdown-scanner :regex (create-scanner
'(:sequence strong-em-1))
:name 'strong-em
:priority 8)
,(make-markdown-scanner :regex (create-scanner
'(:sequence strong-em-2))
:name 'strong-em
:priority 9)
,(make-markdown-scanner :regex (create-scanner '(:sequence strong-2))
:name 'strong
:priority 10)
,(make-markdown-scanner :regex (create-scanner '(:sequence strong-1))
:name 'strong
:priority 11)
,(make-markdown-scanner :regex (create-scanner '(:sequence emphasis-2))
:name 'emphasis
:priority 12)
,(make-markdown-scanner :regex (create-scanner '(:sequence emphasis-1))
:name 'emphasis
:priority 13)
,(make-markdown-scanner :regex (create-scanner '(:sequence auto-link))
:name 'link
:priority 14)
,(make-markdown-scanner :regex (create-scanner '(:sequence auto-mail))
:name 'mail
:priority 15)
,(make-markdown-scanner :regex (create-scanner '(:sequence entity))
:name 'entity
:priority 16)
,(make-markdown-scanner :regex (create-scanner '(:sequence html))
:name 'html
:priority 7.5)
,(make-markdown-scanner
:regex (create-scanner '(:sequence line-ends-with-two-spaces))
:name 'break
:priority 1.8)
)))
(setf (item-at-1 *spanner-parsing-environments* '(code))
(make-instance
'sorted-list-container
:sorter '<
:key 'scanner-priority
:initial-contents
`(,(make-markdown-scanner
:regex (create-scanner '(:sequence html))
:name 'html
:priority 1)
,(make-markdown-scanner
:regex (create-scanner '(:sequence entity))
:name 'entity
:priority 2))))
(defun scanners-for-chunk (chunk)
(let ((it nil))
(cond ((setf it (item-at-1 *spanner-parsing-environments*
(markup-class chunk)))
(values it (markup-class chunk)))
(t
(values (item-at-1 *spanner-parsing-environments* 'default) nil)))))
(defmethod handle-spans ((document abstract-document))
(iterate-elements
(chunks document)
(lambda (chunk)
(handle-spans chunk)))
document)
(defmethod handle-spans ((chunk chunk))
(setf (slot-value chunk 'lines)
(bind ((lines (slot-value chunk 'lines))
((:values scanners kind) (scanners-for-chunk chunk))
(*current-span* kind))
(scan-lines-with-scanners lines scanners)))
chunk)
(defun scan-lines-with-scanners (lines scanners)
(when (or (consp lines)
(typep lines 'cl-containers:iteratable-container-mixin))
(iterate-elements
scanners
(lambda (scanner)
(let ((name (scanner-name scanner)))
(setf lines
(let ((result nil))
(iterate-elements
lines
(lambda (line)
(setf result
(append result (scan-one-span
line name scanner scanners)))))
result))))))
lines)
(defmethod scan-one-span ((line (eql nil)) scanner-name scanner scanners)
(declare (ignorable scanner-name scanner scanners))
(list ""))
(defmethod scan-one-span ((line cons) scanner-name scanner scanners)
;;?? what special case does this handle?
(if (process-span-in-span-p scanner-name (first line))
`((,(first line)
,@(let ((*current-span* (first line)))
(scan-one-span (second line) scanner-name scanner scanners))
,@(nthcdr 2 line)))
(list line)))
(defmethod process-span-in-span-p ((sub-span t) (current-span t))
(values t))
(defmethod process-span-in-span-p
((sub-span (eql nil)) (current-span (eql 'html)))
(values nil))
(defmethod process-span-in-span-p ((sub-span t) (current-span (eql 'html)))
(values nil))
(defmethod process-span-in-span-p ((sub-span (eql 'html)) (current-span t))
(values nil))
(defmethod process-span-in-span-p ((sub-span (eql 'html)) (current-span null))
(values t))
(defmethod process-span-in-span-p
((sub-span (eql 'link)) (current-span (eql 'code)))
(values nil))
(defmethod process-span-in-span-p
((sub-span (eql 'html)) (current-span (eql 'code)))
(values nil))
(defmethod process-span-in-span-p ((sub-span t) (current-span (eql 'code)))
(values nil))
(defmethod process-span-in-span-p
((sub-span t) (current-span (eql 'coded-reference-link)))
(values nil))
(defmethod scan-one-span ((line string) scanner-name scanner scanners)
#+debug
(print (list :sos scanner-name *current-span*
(process-span-in-span-p scanner-name *current-span*)
line))
(let ((found? nil)
(result nil)
(last-e 0)
(regex (scanner-regex scanner))
(scanner-fn (scanner-function scanner)))
(when (process-span-in-span-p scanner-name *current-span*)
(flet ((sub-scan (it)
(let ((*current-span* scanner-name))
(scan-lines-with-scanners it scanners))))
(do-scans (s e gs ge regex line)
(let ((registers (loop for s-value across gs
for e-value across ge
when (and (not (null s-value))
(/= s-value e-value)) collect
(sub-scan (subseq line s-value e-value)))))
(setf registers (process-span scanner-name registers))
(let ((converted
`(,@(when (plusp s)
`(,(sub-scan (subseq line last-e s))))
,(if scanner-fn
(funcall scanner-fn scanner-name registers)
`(,scanner-name ,@registers)))))
(setf found? t
last-e e
result (append result converted)))))
(when found?
(setf result
(let ((last (sub-scan (subseq line last-e))))
(if (plusp (size last))
(append result (list last))
result)))
(return-from scan-one-span
(values (combine-strings result) t))))))
(values (list line) nil))
(defun combine-strings (list)
(let ((result nil)
(current nil))
(flet ((maybe-add (something)
(when something
(setf result (nconc result (list something))))))
(iterate-elements
list
(lambda (elt)
(cond ((stringp elt)
(if current
(setf current (concatenate 'string current elt))
(setf current elt)))
(t
(maybe-add current)
(maybe-add elt)
(setf current nil)))))
(maybe-add current)
result)))
#+(or)
(ensure-same (combine-strings '("a" "b" 23 "c" "d")) ("ab" 23 "cd"))
#+(or)
(ensure-same (combine-strings '("a" "b" 23 2)) ("ab" 23 2))
#+(or)
(ensure-same (combine-strings '(1 2 3 )) (1 2 3))
#+(or)
(defmethod scan-one-span ((line string) scanner-name scanner scanners)
(let ((found? nil)
(result nil)
(last-e 0)
(regex (scanner-regex scanner))
(scanner-fn (scanner-function scanner))
(last-thing nil))
(when (process-span-in-span-p scanner-name *current-span*)
(flet ((sub-scan (it)
(let ((*current-span* scanner-name))
(scan-lines-with-scanners it scanners))))
(do-scans (s e gs ge regex line)
(let ((registers (loop for s-value across gs
for e-value across ge
when (and (not (null s-value))
(/= s-value e-value)) collect
(sub-scan (subseq line s-value e-value)))))
(setf registers (process-span scanner-name registers))
(let ((converted
`(,@(when (plusp s)
`(,(sub-scan (subseq line last-e s))))
,(if scanner-fn
(funcall scanner-fn scanner-name registers)
`(,scanner-name ,@registers)))))
(print (list :c converted last-thing))
(cond ((and (stringp converted)
(stringp last-thing))
(setf (first (last result))
(concatenate 'string last-thing converted)))
(t
(setf result (append result converted))))
(setf found? t
last-e e
last-thing converted))))
(when found?
(return-from scan-one-span
(values (let ((last (sub-scan (subseq line last-e))))
(if (plusp (size last))
(append result (list last))
result))
t))))))
(values (list line) nil))
(defun convert-escape-temporarily (scanner-name registers)
(declare (ignore scanner-name))
(assert (position (aref (first registers) 0) *escape-characters*))
(format nil "~c~c~a~c~c"
#\Null #\Null
(position (aref (first registers) 0) *escape-characters*)
#\Null #\Null))
(defmethod unconvert-escapes ((thing t))
thing)
(defmethod unconvert-escapes ((string string))
(cl-ppcre:regex-replace-all
'(:sequence escape-kludge) string
(lambda (_ &rest registers)
(declare (ignore _)
(dynamic-extent registers))
;(print registers)
(let ((ch (parse-integer (first registers))))
(string (aref *escape-characters* ch))))
:simple-calls t))
(defmethod unconvert-escapes ((thing list))
(collect-elements thing :transform #'unconvert-escapes))
(defmethod unconvert-escapes ((thing chunk))
(setf (slot-value thing 'lines)
(collect-elements (lines thing) :transform #'unconvert-escapes)))
(defmethod unconvert-escapes ((thing abstract-document))
(iterate-elements (chunks thing) #'unconvert-escapes))
cl-markdown-20101006-darcs/dev/docs.lisp 0000644 0001750 0001750 00000045623 11453110206 016307 0 ustar xach xach (in-package #:cl-markdown)
(defun docs-package ()
(let ((property (document-property :docs-package)))
(typecase property
(package property)
(t (let ((package
(or (find-package property)
(find-package (string-upcase property)))))
(setf (document-property :docs-package) package))))))
(defmethod check-exportedp ((symbol cons))
;; setf methods
(check-exportedp (second symbol)))
(defmethod check-exportedp ((symbol symbol))
(unless (or (eq (nth-value 1 (find-symbol (symbol-name symbol)
(symbol-package symbol)))
:external)
(eq (nth-value 1 (find-symbol (symbol-name symbol)
*package*))
:external))
(markdown-warning "Symbol ~s is not exported" symbol)))
(defun ensure-documentation-holder (thing &optional (package nil))
(etypecase thing
(symbol (if (and package (not (eq (symbol-package thing) package)))
(intern (symbol-name thing) package)
thing))
(string (if (and package (find #\space thing))
(let ((*package* package)) (read-from-string thing))
(intern thing package)))
(cons (if (and (eql (first thing) 'setf)
(= (length thing) 2)
(typep (second thing) 'symbol))
(list 'setf (ensure-documentation-holder (second thing) package))
(error "`~a` cannot be converted into something that can hold documentation" thing)))))
(defextension (docs :arguments ((name) (desired-kind))
:insertp t)
(bind ((*package* (or (docs-package) *package*))
symbol)
(labels ((find-docs (thing)
(bind (((:values kinds nil)
(let ((it nil))
(cond ((setf it (symbol-identities-with-docstring
thing desired-kind))
(values it t))
(desired-kind
nil)
(t
(values (mapcar
(lambda (x) (cons x nil))
(symbol-identities thing)) nil))))))
(setf symbol thing)
kinds)))
(bind ((kinds (or (find-docs (ensure-documentation-holder name))
(find-docs (ensure-documentation-holder
name *package*))))
(potentially-ambiguous? (> (length
(symbol-identities-with-docstring
symbol nil))
1))
(kind (or (first kinds) (cons desired-kind nil)))
(docs (and (cdr kind) (find-documentation
symbol (form-keyword (cdr kind)))))
(identity (car kind)))
(ecase phase
(:parse
;;?? could memoize this (where is it stored? in add-docs-item?)
(cond ((> (length kinds) 1)
(markdown-warning "Multiple interpretations found for ~a (~{~a~^, ~}; specify type (using ~a for now)"
name kinds identity))
((null kinds)
(markdown-warning "No docstring found for ~a (package is ~s~@[, kind is ~s~])"
name (package-name (docs-package)) kind))
(t
(check-exportedp symbol)
(add-docs-item symbol identity)
;; this is the result: t if we need to generate an anchor
(documentation-needs-anchor-p name identity))))
(:render
(when (first result)
(anchor-documentation
name identity
(and (not (documentation-needs-anchor-p name nil))
potentially-ambiguous?)))
(render-documentation (form-keyword identity) symbol docs)
nil))))))
(defmethod render-documentation (identity symbol docs)
(format *output-stream*
"
" identity)
(format *output-stream*
"
")
(format *output-stream* "
")
(format *output-stream* "X")
(format *output-stream*
"~&")
(stream-string-for-html (symbol-name symbol) *output-stream*)
(format *output-stream* "")
(when (thing-may-have-arguments-p symbol)
(let ((arguments (mopu:function-arglist symbol)))
(when arguments
(format *output-stream*
"~&")
(display-arguments arguments :kind identity)
(format *output-stream* ""))))
(format *output-stream* "~&
~%")
(format *output-stream*
"~&
~a" identity)
(format *output-stream* "~&
~%")
(unless (document-property :docs-signatures-only)
(format *output-stream*
"
")
(cond
(docs
(markdown docs
:stream *output-stream*
:format *current-format*
:properties '(("html" . nil)
(:omit-final-paragraph . t)
(:omit-initial-paragraph . t))
:document-class 'included-document))
(t
(format
*output-stream*
"No documentation found")))
(format *output-stream* "~&
~%"))
(format *output-stream* "~&
~%"))
(defun documentation-needs-anchor-p (name identity)
"Return true if an anchor is needed and side effects the anchors table
so that it won't return true the next time it is called."
(unless (documentation-anchored-p name identity)
(setf (documentation-anchored-p name identity) t)))
(defun anchor-documentation (name identity &optional potentially-ambiguous?)
(when identity
(output-anchor (docs-link-name name identity)))
(unless potentially-ambiguous?
(output-anchor name)))
(defun docs-link-name (name identity)
(format nil "~a.~a" identity name))
(defun documentation-anchors-table ()
(or (document-property :documentation-anchors)
(setf (document-property :documentation-anchors)
(make-container 'simple-associative-container :test 'equal))))
(defun documentation-anchored-p (name identity)
(item-at-1 (documentation-anchors-table)
(cons name identity)))
(defun (setf documentation-anchored-p) (value name identity)
(setf (item-at-1 (documentation-anchors-table)
(cons name identity))
value))
(defun output-documentation-link (item kind text)
(let ((name (html-safe-name (docs-link-name item kind))))
(format *output-stream*
"~&\~a"
name
(stream-string-for-html (ensure-string text) nil))))
#|
docs-index
look for %items-to-index ==
(item-at-1 (item-at-1 (metadata *current-document*) :docs) kind)
|#
(defextension (docs-index :arguments ((kind-or-kinds :required)
index-kind)
:insertp t)
(setf kind-or-kinds (ensure-list kind-or-kinds))
(unless index-kind
(setf index-kind (first kind-or-kinds)))
(when (eq phase :render)
(bind ((items (%items-to-index kind-or-kinds)))
(cond ((empty-p items)
(markdown-warning
(if (length-1-list-p kind-or-kinds)
"There are no items of kind ~{~a~} documented."
"There are no itmes of kinds~{ ~a~^ or~} documented")
kind-or-kinds))
(t
(output-anchor index-kind)
(format *output-stream*
"~&" index-kind)
(format *output-stream* "~&
")
(loop for (item . real-kind) in items do
(output-documentation-link item real-kind item))
(format *output-stream* "~&
"))))))
(defun canonize-index-kind (kind)
(intern (string-downcase (ensure-string kind))
(load-time-value (find-package :cl-markdown))))
(defun %items-to-index (kinds)
(let ((docs (item-at-1 (metadata *current-document*) :docs))
(kinds (mapcar #'canonize-index-kind kinds)))
(sort
(cond ((member 'all kinds)
(let ((result nil))
(iterate-key-value
docs
(lambda (kind item-table)
(setf result
(nconc result
(collect-keys
item-table
:transform (lambda (symbol)
(cons symbol kind)))))))
result))
(t
(loop for kind in kinds nconc
(collect-keys (item-at-1 docs kind)
:transform (lambda (symbol)
(cons symbol kind))))))
#'string-lessp
:key #'first)))
(defun add-docs-item (thing kind)
(let ((kind (canonize-index-kind kind)))
(add-docs-link thing kind)
(bind ((docs-items
(or (item-at-1 (metadata *current-document*) :docs)
(setf (item-at-1 (metadata *current-document*) :docs)
(make-container 'simple-associative-container))))
(kind-items
(or (item-at-1 docs-items kind)
(setf (item-at-1 docs-items kind)
(make-container 'simple-associative-container)))))
(setf (item-at-1 kind-items thing) t))))
(defun add-docs-link (thing kind)
(let ((kind (canonize-index-kind kind)))
(flet ((add-link (name title)
(let ((anchor (html-safe-name (ensure-string name))))
(setf (item-at (link-info *current-document*) name)
(make-instance 'link-info
:id name :url (format nil "#~a" anchor)
:title title)))))
(add-link (docs-link-name thing kind)
(format nil "description of ~a ~a" kind thing))
(bind ((kinds (symbol-identities-with-docstring thing)))
(when (length-1-list-p kinds)
(add-link (format nil "~a" thing)
(format nil "description of ~a" thing)))))))
(defmethod thing-may-have-arguments-p ((symbol symbol))
(and (fboundp symbol)
(or
(typep (symbol-function symbol) 'function)
(macro-function symbol)
(typep (symbol-function symbol) 'standard-generic-function))))
(defmethod thing-may-have-arguments-p ((list list))
(thing-names-setf-function-p list))
#|
If there is a suppliedp variable for an optional or keyword arg, it should
not be printed.
If there is an initial value for an optional or keyword, it should be
printed with ~S otherwise 'foo :foo "foo" all print as foo in the doc.
If the initial value is a big expression, it should not print in the doc.
The effect of that big expression should be explained in words.
If the initial value is nil, it does not need to show in the argument line.
|#
#|
(defun xxxfoo (triple-count index-count
&key (skip-size (ag-property 'default-metaindex-skip-size))
(unique-strings (floor triple-count 3))
(average-string-size nil))
)
(let ((*output-stream* *standard-output*))
(display-arguments (mopu:function-arglist 'lift:ensure-cases) :kind 'macro))
(trace display-arguments)
(untrace display-arguments)
|#
(defun display-arguments (arguments &key (kind nil))
;; kind can be anything returned by symbol-identities
;; currently, we only care about macros
(let ((space-entity (document-property "docs-space-entity" " "))
(first? t)
(stream *output-stream*))
(dolist (argument arguments)
;; bail on &aux
(when (and (symbolp argument)
(string-equal (symbol-name argument) "&aux"))
(return))
(unless first?
(format stream "~a" space-entity))
;; dispatch?
(cond ((consp argument)
(case kind
(:macro
(format stream "(")
(display-arguments argument :kind kind)
(format stream ")"))
(t
(cond ((eq (length argument) 3)
;; (name initform supplied)
;; ((:key name) initform supplied)
;; just recur without the supplied
(display-arguments (list (butlast argument)) :kind kind))
((eq (length argument) 2)
(if (consp (car argument))
;; ((:key name) initform)
(bind ((((key name) initform) argument)
(package (symbol-package name))
(new-name (intern (symbol-name key) package)))
(display-arguments (list (list new-name initform))
:kind kind))
;; (name initform)
(bind (((name initform) argument)
(name (stream-string-for-html
(ensure-string name) nil)))
(cond ((null initform)
;; just show argument
(format stream "~(~a~)" name))
((constantp initform)
;; show both
(format stream "~((~a ~s)~)"
name initform))
(t
;; just show name
(format stream "~(~a~)" name))))))
(t
;; probably part of a macro
(format stream "(")
(display-arguments argument :kind kind)
(format stream ")"))))))
((and (symbolp argument)
(string-equal (symbol-name argument) "&"
:start1 0 :start2 0 :end1 1 :end2 1))
(format stream
"~(~a~)"
(stream-string-for-html (symbol-name argument) nil)))
(t
(format stream "~(~a~)"
(stream-string-for-html (symbol-name argument) nil))))
(setf first? nil))))
(defgeneric find-documentation (thing strategy)
(:documentation "Return the documentation for thing using strategy. The default is to call the Common Lisp documentation method with strategy being used as the type."))
(defmethod find-documentation (thing strategy)
(documentation thing (intern (symbol-name strategy)
(load-time-value (find-package :common-lisp)))))
(defmethod find-documentation (thing (strategy (eql :setf)))
(documentation (second thing) 'setf))
(defmethod find-documentation (thing (strategy (eql :function)))
(cond ((and (fboundp thing)
(typep (symbol-function thing) 'standard-generic-function))
(let ((docstring (call-next-method))
(strings
(loop for m in (mopu:generic-function-methods
(symbol-function thing))
when (documentation m 'function) append
(list (documentation m 'function)))))
(format nil "~@[~a~]~:[~;~%~%~]~@[~{~a~^~%~%~}~]"
docstring (and docstring strings) strings)))
(t
(call-next-method))))
(defparameter *symbol-identities*
'((thing-names-class-p class type)
(thing-names-condition-p condition nil)
(thing-names-constant-p constant variable)
(thing-names-function-p function nil)
;; FIXME - for now, we don't separate 'em
(thing-names-generic-function-p function function)
(thing-names-macro-p macro function)
(thing-names-setf-function-p setf)
(thing-names-slot-accessor-p function function)
(thing-names-structure-p structure structure)
(thing-names-type-p type type structure)
(thing-names-variable-p variable nil)))
(defun add-documentation-strategy (test thing strategy)
(pushnew (list test thing strategy) *symbol-identities*
:test #'equal
:key #'first))
(defun kind-mappings (kind)
"Some kinds of things have their docstrings in 'other' places. For example,
macros put their docstrings under 'function. This function papers over the
distinction."
(case (form-keyword kind)
((:macro :generic-function) '(function))
(:class '(type))
(:condition '(type))
(:structure '(structure))
(:constant '(variable))
(:type '(type))
(t kind)))
;; FIXME - fully reconcile list of docstring with list of identities
(defun symbol-identities-with-docstring (symbol &optional expected-kind)
(let ((kinds
(loop for kind in
(ensure-list
(or (and expected-kind
(ensure-documentation-holder expected-kind))
(symbol-identities symbol)))
for mappings = (kind-mappings kind)
for docs = nil
when
(or (and (atom mappings)
(find-documentation symbol (form-keyword mappings))
(setf docs mappings))
(and (consp mappings)
(some (lambda (doc-kind)
(and (find-documentation
symbol (form-keyword doc-kind))
(setf docs doc-kind)))
mappings))) collect
(cons kind docs))))
;; FIXME -- I've got that adhoc feeling... priority?
(when (find 'macro kinds :key #'car)
(setf kinds (delete 'function kinds :key #'car)))
(when (find 'constant kinds :key #'car)
(setf kinds (delete 'variable kinds :key #'car)))
(delete-duplicates kinds :test 'eq :key #'first)))
(defun symbol-identities (symbol)
;; cf. *symbol-identities*
(delete-duplicates
(loop for (predicate kind nil) in *symbol-identities*
when (funcall predicate symbol) collect kind)))
(defun thing-names-class-p (doc-holder)
(and (symbolp doc-holder)
(let ((class (find-class doc-holder nil)))
(and class
(typep class 'standard-class)
(not (conditionp class))))))
;; FIXME -- this (most likely) won't work on Lisps that don't use CLOS
;; for conditions
(defun conditionp (thing)
"Returns true if and only if thing is a condition"
(mopu:subclassp thing 'condition))
(defun thing-names-condition-p (doc-holder)
(let ((it nil))
(and (setf it (symbolp doc-holder))
(setf it (find-class doc-holder nil))
(setf it (conditionp it)))))
(defun thing-names-constant-p (doc-holder)
(and (symbolp doc-holder)
(boundp doc-holder)
(constantp doc-holder)))
(defun thing-names-function-p (doc-holder)
(and (symbolp doc-holder)
(fboundp doc-holder)
(not (macro-function doc-holder))
(typep (symbol-function doc-holder) 'function)
(not (typep (symbol-function doc-holder) 'standard-generic-function))))
(defun thing-names-generic-function-p (doc-holder)
(and (symbolp doc-holder)
(fboundp doc-holder)
(typep (symbol-function doc-holder) 'standard-generic-function)
(some (lambda (m)
(not (or (mopu:reader-method-p m)
(mopu:writer-method-p m))))
(mopu:generic-function-methods (symbol-function doc-holder)))))
(defun thing-names-macro-p (doc-holder)
(and (symbolp doc-holder)
(macro-function doc-holder)))
(defun thing-names-setf-function-p (doc-holder)
(and (consp doc-holder)
(eql (first doc-holder) 'setf)
(length-exactly-p doc-holder 2)
(symbolp (second doc-holder))
(ignore-errors (eval `(function ,doc-holder)))))
(defun thing-names-slot-accessor-p (doc-holder)
(and (symbolp doc-holder)
(fboundp doc-holder)
(typep (symbol-function doc-holder) 'standard-generic-function)
(some (lambda (m)
(or (mopu:reader-method-p m)
(mopu:writer-method-p m)))
(mopu:generic-function-methods (symbol-function doc-holder)))))
(defun thing-names-structure-p (doc-holder)
(and (symbolp doc-holder)
(let ((class (find-class doc-holder nil)))
(and class
(typep class 'structure-class)))))
(defun thing-names-variable-p (doc-holder)
(and (symbolp doc-holder)
(boundp doc-holder)
(not (constantp doc-holder))))
(defun thing-names-type-p (doc-holder)
(and (symbolp doc-holder)
(not (thing-names-class-p doc-holder))
(not (thing-names-condition-p doc-holder))
#+allegro
(or (excl:normalize-type
doc-holder :loud (lambda (&rest r)
(declare (ignore r))
(return-from thing-names-type-p nil)))
t)))
;;;;
(defextension (links-list :arguments ((kind-or-kinds :required)
index-kind))
(setf kind-or-kinds (ensure-list kind-or-kinds))
(unless index-kind
(setf index-kind (first kind-or-kinds)))
(when (eq phase :render)
(bind ((items (%items-to-index kind-or-kinds)))
(cond ((empty-p items)
(markdown-warning
(if (length-1-list-p kind-or-kinds)
"There are no items of kind ~{~a~} documented."
"There are no itmes of kinds~{ ~a~^ or~} documented")
kind-or-kinds))
(t
(format *output-stream*
"~&(" index-kind)
(loop for (item . real-kind) in items do
(output-links-list-item item real-kind item)))))))
(defun output-links-list-item (item kind text)
(let ((name (html-safe-name (format nil "~a.~a" item kind))))
(format *output-stream*
"~&(#~a ~a)"
name text)))
#|
(defvar *foo-1* 2)
(defun foo-1 ()
"foo-1"
*foo-1*)
(defun inv-foo-1 (value)
"inv-foo-1"
(setf *foo-1* value))
(defsetf foo-1 inv-foo-1 "defsetf foo-1")
(defun (setf foo-1) (value)
"(setf foo-1)"
(setf *foo-1* value))
(documentation 'foo-1 'setf)
|# cl-markdown-20101006-darcs/dev/plain.lisp 0000644 0001750 0001750 00000004200 11453110206 016444 0 ustar xach xach (in-package #:cl-markdown)
#|
(markdown "`test` blue **beans**" :format :plain)
(markdown "Eta
* beta
* data
Eta" :format :plain)
|#
(defmethod render ((document abstract-document) (style (eql :plain)) stream)
(declare (ignore stream))
(render-plain document))
(defmethod render-plain ((document abstract-document))
(bind ((current-chunk nil))
(labels ((render-block (block level markup inner?)
(declare (ignore markup))
(let ((add-markup? (not (eq (first block) current-chunk))))
(cond ((or (length-1-list-p block))
(render-plain (first block)))
((not add-markup?)
(render-plain (first block))
(do-it (rest block) level))
(t
(setf current-chunk (and inner? (first block)))
(do-it block level)))))
(do-it (chunks level)
(loop for rest = chunks then (rest rest)
for chunk = (first rest) then (first rest)
while chunk
for new-level = (level chunk)
when (= level new-level) do
(let ((index (inner-block rest))
(inner-markup (html-inner-block-markup chunk)))
(render-block (subseq rest 0 index)
level inner-markup t)
(setf rest (nthcdr (1- index) rest)))
when (< level new-level) do
(multiple-value-bind (block remaining method)
(next-block rest new-level)
(declare (ignore method))
(render-block
block new-level (html-block-markup chunk) nil)
(setf rest remaining)))))
(do-it (collect-elements (chunks document)) (level document)))))
(defmethod render-plain ((chunk chunk))
(bind ((paragraph? (paragraph? chunk)))
(iterate-elements
(lines chunk)
(lambda (line)
(render-plain line)))
(when paragraph?
(fresh-line *output-stream*))))
(defmethod render-plain ((line string))
(format *output-stream* "~a" line))
(defmethod render-plain ((chunk list))
(render-span-plain (first chunk) (rest chunk)))
(defmethod render-span-plain ((code t) body)
(format *output-stream* "~a" (first body)))
(defmethod render-span-plain ((code (eql 'eval)) body)
(render-handle-eval body))
(defmethod render-span-plain ((code (eql 'code-eval)) body)
(render-handle-eval body))
cl-markdown-20101006-darcs/dev/api.lisp 0000644 0001750 0001750 00000003071 11453110206 016117 0 ustar xach xach (in-package #:cl-markdown)
(defgeneric reset (thing)
)
(defgeneric (setf document-property) (value name))
(defgeneric render-to-stream (document style stream-specifier)
)
(defgeneric main-parent (document)
)
(defgeneric handle-spans (document)
)
(defgeneric scan-one-span (what scanner-name scanner scanners)
)
(defgeneric process-span-in-span-p (sub-span current-span)
)
(defgeneric unconvert-escapes (what)
)
(defgeneric render (document style stream)
)
(defgeneric it-starts-with-block-level-html-p (chunk)
)
(defgeneric markup-class-mergable-p (what)
)
(defgeneric merge-lines-in-chunks (what)
)
(defgeneric can-merge-lines-p (first second)
)
(defgeneric handle-paragraph-eval-interactions (what)
)
(defgeneric encode-html (what encoding-method &rest codes)
)
(defgeneric markup-class-for-html (what)
)
(defgeneric render-span-to-html (kind body encoding-method)
)
(defgeneric generate-link-output (link-info text)
)
(defgeneric add-html-header-p (document)
)
(defgeneric render-plain (what)
)
(defgeneric render-span-plain (kind body)
)
(defgeneric process-span-for (kind command args)
)
(defgeneric generate-link-output-for-kind (kind link-info text)
)
(defgeneric process-span (name registers)
(:documentation "Called during span processing on each match of name in the
document. Registers a list of the registers captured by names regular expression.
Returns a possibly new set of registers.")
(:method ((name t) (registers t))
(values registers)))
(defgeneric print-html-markup (markup stream)
)
cl-markdown-20101006-darcs/dev/markdown.lisp 0000644 0001750 0001750 00000103147 11453110206 017175 0 ustar xach xach (in-package #:cl-markdown)
(defun markdown (source &key (stream *default-stream*)
(format *default-format*)
(additional-extensions nil)
(render-extensions nil)
(parse-extensions nil)
(properties nil)
(parent *current-document*)
(document-class 'document)
)
"Convert source into a markdown document object and optionally render it to stream using format. Source can be either a string or a pathname or a stream. Stream is like the stream argument in format; it can be a pathname or t \(short for *standard-output*\) or nil \(which will place the output into a string\). Format can be :html or :none. In the latter case, no output will be generated.
The markdown command returns \(as multiple values\) the generated document object and any return value from the rendering \(e.g., the string produced when the stream is nil\)."
;; we chunk-source, run post-processor, handle-spans, cleanup and then render
(let ((*current-document* (make-container document-class
:parent parent
:source source))
(*render-active-functions*
(mapcar #'canonize-command
(or render-extensions
(if additional-extensions
`(,@additional-extensions ,@*render-active-functions*)
*render-active-functions*))))
(*parse-active-functions*
(mapcar #'canonize-command
(or parse-extensions
(if additional-extensions
`(,@additional-extensions ,@*parse-active-functions*)
*parse-active-functions*))))
(*default-pathname-defaults* (or (and (typep source 'pathname)
(containing-directory source))
*default-pathname-defaults*)))
;; pull in properties
(loop for (name . value) in properties do
(setf (document-property name) value))
(chunk-source *current-document* source)
(iterate-elements
(chunk-post-processors *parsing-environment*)
(lambda (processor)
(funcall processor *current-document*)))
(handle-spans *current-document*)
(cleanup *current-document*)
(values *current-document*
(render-to-stream *current-document* format stream))))
(defun containing-directory (pathspec)
"Return the containing directory of the thing to which pathspac points. For example:
(containing-directory \"/foo/bar/bis.temp\")
\"/foo/bar/\"
> (containing-directory \"/foo/bar/\")
\"/foo/\"
"
(make-pathname
:directory `(,@(butlast (pathname-directory pathspec)
(if (directory-pathname-p pathspec) 1 0)))
:name nil
:type nil
:defaults pathspec))
(defmethod render :before
((document abstract-document) style stream)
(declare (ignore style))
(when (typep stream 'file-stream)
(setf (destination document) (pathname stream))))
(defmethod render ((document abstract-document) (style (eql :none)) stream)
(declare (ignore stream))
nil)
(defmethod reset ((env parsing-environment))
(empty! (chunk-parsing-environment env))
(insert-item (chunk-parsing-environment env)
(item-at-1 *chunk-parsing-environments* 'toplevel))
(setf (chunk-level env) 0
(current-strip env) "")
(setf (chunk-post-processors env)
(list
'handle-link-reference-titles
'handle-extended-link-references
'handle-code ; before hr and paragraphs
'handle-paragraphs ; before headers
'handle-setext-headers ; before hr
'handle-horizontal-rules ; before bullet lists, after code
'handle-bullet-lists
'handle-number-lists
'handle-bullet-paragraphs
'handle-blockquotes
'handle-atx-headers
'merge-chunks-in-document
'merge-lines-in-chunks
'canonize-document))
(empty! (line-code->stripper env))
(empty! (strippers env))
(setf (item-at-1 (line-code->stripper env) 'line-is-blockquote-p)
'blockquote-stripper
(item-at-1 (line-code->stripper env) 'line-starts-with-bullet-p)
'one-tab-stripper
(item-at-1 (line-code->stripper env) 'line-is-code-p)
'one-tab-stripper
(item-at-1 (line-code->stripper env) 'line-starts-with-number-p)
'one-tab-stripper))
(defun null-stripper (line)
(values line t))
(defun one-tab-stripper (line)
(let ((indentation 0)
(index 0))
(loop for ch across line
while (< indentation *spaces-per-tab*) do
(incf index)
(cond ((char= ch #\ )
(incf indentation))
((char= ch #\Tab)
(incf indentation *spaces-per-tab*))
(t
(return))))
(if (>= indentation *spaces-per-tab*)
(values (subseq line index) t)
(values line nil))))
;;?? Gary King 2006-01-23: yerch, I don't like it either...
(defun blockquote-stripper (line)
"Strips putative initial blockquote and up to 3 spaces"
(let ((count 0)
(found-bq? nil))
(cond ((>= (blockquote-count line) 1)
(loop repeat *spaces-per-tab*
for ch across line
while (and (not found-bq?)
(or (char-equal ch #\ )
(and (char-equal ch #\>)
(setf found-bq? t)))) do
(incf count))
(cond ((not (null found-bq?))
(when (and (> (size line) (1+ count))
(char-equal (aref line count) #\ ))
(incf count))
(values (subseq line count) t))
(t
(values line nil))))
(t
(values line nil)))))
(defun blockquote-count (line)
(let ((count 0))
(loop for ch across line
while (or (char-equal ch #\ )
(char-equal ch #\>))
when (char-equal ch #\>) do (incf count))
count))
(defun line-indentation (line)
(let ((count 0))
(or (loop for ch across line do
(cond ((char-equal ch #\ ) (incf count))
((char-equal ch #\tab) (incf count *spaces-per-tab*))
(t (return count))))
;; empty line
(values 0))))
(defun line-changes-indentation-p (line)
(let ((count 0))
(loop for ch across line do
(cond ((char-equal ch #\ ) (incf count))
((char-equal ch #\tab) (incf count *spaces-per-tab*))
(t (return count))))
(unless (= *current-indentation-level* count)
(setf *current-indentation-level* count)
(values t))))
(defun line-starts-with-bullet-p (line)
;; a bullet and at least one space or tab after it
(let* ((count 0)
(bullet? (loop repeat (1- *spaces-per-tab*)
for ch across line
when (or (char-is-tab-or-space-p ch)
(char-is-bullet-p ch))
do (incf count)
when (char-is-bullet-p ch) do (return t))))
(or (and bullet?
(> (length line) count)
(char-is-tab-or-space-p (aref line count)))
(and (not bullet?)
(> (length line) (1+ count))
(char-is-bullet-p (aref line count))
(char-is-tab-or-space-p (aref line (1+ count)))))))
(defun char-is-tab-or-space-p (ch)
(or (char-equal ch #\ ) (char-equal ch #\Tab)))
(defun char-is-bullet-p (ch)
(or (char-equal ch #\*)
(char-equal ch #\-)
(char-equal ch #\+)))
(defun line-starts-with-number-p (line)
;; at least one digit, then digits and then a period, then a space
;; FIXME -- (Why don't I use a regex?)
(let* ((count 0)
(number? (loop repeat (1- *spaces-per-tab*)
for ch across line
when (or (char-is-tab-or-space-p ch)
(digit-char-p ch))
do (incf count)
when (digit-char-p ch) do (return t))))
(or (and number?
(> (length line) (1+ count))
(char-equal (aref line count) #\.)
(whitespacep (aref line (1+ count))))
;; this is in line-starts-with-bullet-p but looks wacked
#+(or)
(and (not bullet?)
(> (length line) (1+ count))
(char-is-bullet-p (aref line count))
(char-is-tab-or-space-p (aref line (1+ count)))))))
(defun line-is-empty-p (line)
(every-element-p line #'metatilities:whitespacep))
(defun line-is-not-empty-p (line)
(not (line-is-empty-p line)))
(defun line-is-blockquote-p (line)
(unless (line-is-code-p line)
(let ((trimmed-line (string-left-trim '(#\ ) line)))
(and (plusp (size trimmed-line))
(char-equal (aref trimmed-line 0) #\>)))))
(defun line-is-code-p (line)
(>= (line-indentation line) *spaces-per-tab*))
(defun line-could-be-header-marker-p (line)
(and
(or (string-starts-with line "-")
(string-starts-with line "="))
(let ((first-ch (item-at line 0))
(found-ws-p nil))
(every-element-p line (lambda (ch)
(cond (found-ws-p (whitespacep ch))
((whitespacep ch) (setf found-ws-p t))
(t (char= ch first-ch))))))))
(defun line-is-link-label-p (line)
(scan (load-time-value (ppcre:create-scanner '(:sequence link-label))) line))
(defun line-is-extended-link-label-p (line)
(scan (load-time-value
(ppcre:create-scanner '(:sequence extended-link-label))) line))
(defun line-other-p (line)
(declare (ignore line))
;; catch all
(values t))
(defun horizontal-rule-char-p (char)
(member char '(#\- #\* #\_) :test #'char-equal))
(defun line-is-horizontal-rule-p (line)
(let ((amatch nil)
(count 0)
(possible-hr? nil))
(loop for char across line do
(cond ((whitespacep char)
;; ignore
)
((or (and amatch (char-equal amatch char))
(and (not amatch) (horizontal-rule-char-p char)))
(setf amatch char)
(incf count)
(when (>= count *horizontal-rule-count-threshold*)
(setf possible-hr? t)))
(t
(setf possible-hr? nil)
(return))))
(values possible-hr?)))
(setf (item-at-1 *chunk-parsing-environments* 'toplevel)
(make-instance
'chunk-parsing-environment
:name 'toplevel
:line-coders '(line-is-empty-p
line-is-link-label-p
line-is-extended-link-label-p
line-is-code-p
line-is-blockquote-p
line-could-be-header-marker-p
line-is-horizontal-rule-p
line-starts-with-bullet-p
line-starts-with-number-p
line-is-not-empty-p
line-other)
:chunk-enders '(line-is-empty-p
line-starts-with-number-p
line-starts-with-bullet-p
line-is-horizontal-rule-p
line-is-blockquote-p
line-is-link-label-p ; we'll grab title later...
line-is-extended-link-label-p
line-could-be-header-marker-p
atx-header-p
)
:chunk-starters '(line-could-be-header-marker-p
atx-header-p
line-is-not-empty-p )
:parser-map '((line-is-code-p code))))
(setf (item-at-1 *chunk-parsing-environments* 'line-is-code-p)
(make-instance
'chunk-parsing-environment
:name 'code
:chunk-enders '()
:chunk-starters '()))
(defun maybe-strip-line (line)
(bind ((env *parsing-environment*)
(levels 0)
(stripped? nil)
;;?? rather gross, but we don't have reverse iterators yet
(strippers (reverse (collect-elements (strippers env)))))
(block stripping
(iterate-elements
strippers
(lambda (stripper)
(setf (values line stripped?) (funcall stripper line))
(unless stripped?
(return-from stripping))
(incf levels))))
(values line levels)))
(defun chunk-source (document source)
(let* ((result document)
(current nil)
(current-code nil)
(previous-stripper nil)
(level 0)
(old-level level)
(first? 'start-of-document)
(was-blank? nil)
(been-blank? nil)
(just-started? nil)
(*default-pathname-defaults* (or (and (typep source 'pathname)
(containing-directory source))
*default-pathname-defaults*))
(line-iterator nil))
(declare (special line-iterator))
(reset *parsing-environment*)
(labels ((code-line (line)
(values (some-element-p (line-coders (current-chunk-parser))
(lambda (p) (funcall p line)))
(some-element-p
(chunk-enders (current-chunk-parser))
(lambda (p) (funcall p line)))
(some-element-p
(chunk-starters (current-chunk-parser))
(lambda (p) (funcall p line)))))
(end-chunk-p (line code starter ender)
;; End when we have a current chunk AND either
;; the new level is bigger OR there is an ender OR
;; the new level is smaller AND the line isn't empty
(declare (ignore starter))
(when (and current
(or (> level old-level)
(and (< level old-level)
(not (line-is-empty-p line)))
ender))
;; special case for hard returns; don't end when the
;; starter is 'line-is-not-empty-p unless it is preceeded
;; by a blank line
(and ;(not (eq code 'line-is-empty-p))
(or (not (eq code 'line-is-not-empty-p))
(and (eq code 'line-is-not-empty-p)
was-blank?)))))
(chunk-line (line)
(setf (values line level) (maybe-strip-line line))
(unless (line-is-empty-p line)
(loop repeat (- old-level level)
while (> (size (chunk-parsing-environment
*parsing-environment*)) 1) do
(pop-item (chunk-parsing-environment
*parsing-environment*))))
(bind (((:values code ender starter) (code-line line)))
#+(or)
(format
t "~%~S~% L/OL: ~d/~d [~a] ~& C: ~A E: ~A S: ~A N: ~a X: ~d~@[ L: ~d~] WB: ~a, BB: ~a"
line level old-level
;(size (strippers *parsing-environment*))
(collect-elements (strippers *parsing-environment*))
code ender starter (name (current-chunk-parser))
(size (chunk-parsing-environment *parsing-environment*))
(and current (size (lines current)))
was-blank? been-blank?)
;; End current chunk?
(when (end-chunk-p line code starter ender)
#+(or)
(format t " (c: ~a e: ~a) --> end~%" code ender)
(setf (ended-by current) code
(blank-line-after? current) (line-is-empty-p line))
(insert-item (chunks result) current)
(setf previous-stripper (stripper? current))
(setf current nil))
(setf current-code code)
;; deal with embedded brackets
(when (and (not (eq code 'line-is-code-p))
(or
(not current)
(not (eq (started-by current) 'line-is-code-p))))
(setf line (process-brackets document line line-iterator)))
;; Start new chunk?
(awhen (and (not current) starter)
#+(or)
(format t " --> start")
(let ((stripper
(item-at-1 (line-code->stripper *parsing-environment*)
current-code)))
(setf level (+ level (if stripper 1 0))
current (make-instance
'chunk
:started-by (or current-code first?)
:blank-line-before? (and (not first?)
was-blank?)
:indentation (line-indentation line)
:level level
:stripper? stripper)
first? nil
(chunk-level *parsing-environment*) level)
;; if there is a new stripper, use it
(when stripper
(setf line (funcall stripper line)))
(when (and (>= level old-level) stripper)
(when (and previous-stripper (= level old-level))
(pop-item (strippers *parsing-environment*)))
(insert-item (strippers *parsing-environment*) stripper)
))
(setf just-started? t))
;; add to current chunk
(when current
(if (line-is-empty-p line)
(insert-item (lines current) "")
(insert-item (lines current)
;; consing is fun
;; FIXME - OK, but why do I do this?!
;; A: make sure every line ends with a space.
;; Q: duh, but why?!
#+(or)
line
(if (and (plusp (length line))
(char= (aref line (1- (length line)))
#\Space))
line
(concatenate 'string line " "))))
)
(loop while (> level old-level) do
(insert-item
(chunk-parsing-environment *parsing-environment*)
(or
(item-at-1 *chunk-parsing-environments* code)
(item-at-1 *chunk-parsing-environments* 'toplevel)))
(incf old-level))
(when (and been-blank?
(not (line-is-empty-p line)))
(setf old-level level)
(loop while (> (size (strippers *parsing-environment*))
level) do
(pop-item (strippers *parsing-environment*))))
(setf was-blank? (line-is-empty-p line))
(setf been-blank? (unless just-started?
(or been-blank? was-blank?)))
(setf just-started? nil)
)))
(with-iterator (i source :treat-contents-as :lines
:skip-empty-chunks? nil)
(let ((line-iterator i))
(declare (special line-iterator))
(iterate-elements
line-iterator (lambda (line) (chunk-line line))))))
;; final processing
(cond (current
;; Grab last chunk if any
(setf (ended-by current) 'end-of-document)
(insert-item (chunks result) current))
((not (empty-p (chunks result)))
(let ((last (last-item (chunks result))))
(setf (blank-line-after? last) t)
(when (eq (ended-by last) 'line-is-empty-p)
;; maybe fix ended-by of last chunk
(setf (ended-by last) 'end-of-document)))))
(values result)))
;;; post processors
(defun handle-horizontal-rules (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (or (eq (started-by chunk) 'line-is-horizontal-rule-p)
#+(or)
;; no -- case 12
(eq (ended-by chunk) 'line-is-horizontal-rule-p))
(empty! (lines chunk))
(setf (markup-class chunk) '(horizontal-rule))))))
(defmethod it-starts-with-block-level-html-p ((chunk chunk))
(and (not (empty-p (lines chunk)))
(it-starts-with-block-level-html-p (first-element (lines chunk)))))
;; FIXME - use an r.e., gosh durn it.
(defmethod it-starts-with-block-level-html-p ((line string))
(and (> (length line) 2)
(char= (aref line 0) #\<)
(let* ((pos-> (position #\> line :test 'char=))
(pos-space (position #\Space line :test 'char=))
(pos (and pos-> (or (and pos-space (min pos-space pos->)) pos->)))
(code (and pos (subseq line 1 pos))))
(when (and code (> (length code) 0) (char= (aref code 0) #\/))
(setf code (subseq code 1)))
(and code (find code *block-level-html-tags*
:test 'string-equal)))))
(defun handle-paragraphs (document)
(let ((first? t))
(flet ((blank-before-p (chunk)
(or (blank-line-before? chunk)
(and
;;?? probably a hack
(or (eq (started-by chunk) 'start-of-document)
first?)
(not (document-property :omit-initial-paragraph nil)))))
(blank-after-p (chunk)
(or (blank-line-after? chunk)
(and
;;?? probably a hack
(eq (ended-by chunk) 'end-of-document)
(not (document-property :omit-final-paragraph nil))))))
(iterate-elements
(chunks document)
(lambda (chunk)
(setf (paragraph? chunk)
(and
(not (it-starts-with-block-level-html-p chunk))
(not (member 'code (markup-class chunk)))
(or (and (blank-before-p chunk)
(blank-after-p chunk))
(and (or (blank-before-p chunk)
(blank-after-p chunk))
(not (member (started-by chunk)
'(line-starts-with-bullet-p
line-starts-with-number-p)))))))
(setf first? nil))))))
(defun handle-bullet-paragraphs (document)
;; if I have the heuristic right, a list item only gets a paragraph
;; if is following (preceeded) by another list item and there is a blank
;; line separating them.
(let ((first? t))
(labels
((blank-before-p (chunk)
(or (blank-line-before? chunk)
(and
;;?? probably a hack
(or (eq (started-by chunk) 'start-of-document)
first?)
(not (document-property :omit-initial-paragraph nil)))))
(blank-after-p (chunk)
(or (blank-line-after? chunk)
(and
;;?? probably a hack
(= (level chunk) 1)
(eq (ended-by chunk) 'end-of-document)
(not (document-property :omit-final-paragraph nil)))))
(list-item-p (chunk)
(member (started-by chunk)
'(line-starts-with-bullet-p
line-starts-with-number-p)))
(handle-triple (a b c)
(cond ((and (eq a b) (eq b c)
(list-item-p a))
;; all same
(setf (paragraph? a) nil))
((not (and (eq a b) (eq b c)))
(when (and (list-item-p b))
;; (print (list a b c))
(setf (paragraph? b)
(or (and (list-item-p a)
(= (level a) (level b))
(blank-before-p b)
:bullet-before)
(and (list-item-p c)
(= (level b) (level c))
(blank-after-p b)
:bullet-after))))))
(setf first? nil)))
(map-window-over-elements
(chunks document) 3 1
(lambda (triple)
(bind (((a b c) triple))
(handle-triple a b c)))
:duplicate-ends? t))))
(defun handle-atx-headers (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (and (eq (started-by chunk) 'line-is-not-empty-p)
(atx-header-p (first-element (lines chunk))))
(make-header
chunk (atx-header-markup-class (first-element (lines chunk))))
(setf (first-element (lines chunk))
(remove-atx-header (first-element (lines chunk))))))))
(defun make-header (chunk markup-class)
(push markup-class (markup-class chunk))
(setf (paragraph? chunk) nil))
(defun atx-header-p (line)
(let ((first-non-hash (position-if
(lambda (ch) (not (char-equal ch #\#))) line)))
(and first-non-hash
(< 0 first-non-hash 7))))
(defun atx-header-markup-class (line)
(let ((first-non-hash (position-if
(lambda (ch) (not (char-equal ch #\#))) line)))
(case first-non-hash
(1 'header1)
(2 'header2)
(3 'header3)
(4 'header4)
(5 'header5)
(6 'header6)
(t (error "Unable to determine ATX header class of ~A" line)))))
(defun remove-atx-header (line)
(string-trim '(#\ ) (string-trim '(#\#) line)))
(defun can-merge-chunks-p (chunk1 chunk2)
(and (= (level chunk1) (level chunk2))
(equal (markup-class chunk1) (markup-class chunk2))
(markup-class-mergable-p (markup-class chunk2))
(not (paragraph? chunk2))
(not (eq (ended-by chunk1) 'line-is-empty-p))))
(defmethod markup-class-mergable-p ((markup-class cons))
(every #'markup-class-mergable-p markup-class))
(defmethod markup-class-mergable-p ((markup-class symbol))
(member markup-class '(code quote)))
(defun merge-chunks-in-document (document)
(let ((chunks (make-iterator (chunks document)))
(gatherer nil))
(cl-containers::iterate-forward
chunks
(lambda (chunk)
(if (and gatherer (can-merge-chunks-p gatherer chunk))
(merge-chunks gatherer chunk)
(setf gatherer chunk)))))
(removed-ignored-chunks? document))
(defun merge-chunks (c1 c2)
(iterate-elements (lines c2) (lambda (l) (insert-item (lines c1) l)))
(setf (ignore? c2) t))
(defmethod merge-lines-in-chunks ((document abstract-document))
(iterate-elements
(chunks document)
#'merge-lines-in-chunks))
(defmethod merge-lines-in-chunks ((chunk chunk))
(unless (member 'code (markup-class chunk))
(setf (slot-value chunk 'lines)
(merge-lines-in-chunks (lines chunk)))))
(defmethod merge-lines-in-chunks ((lines iteratable-container-mixin))
(let ((iterator (make-iterator lines))
(gatherer "")
(result nil))
(iterate-forward
iterator
(lambda (line)
(cond ((can-merge-lines-p gatherer line)
(let ((length (length gatherer)))
(cond ((zerop length)
(setf gatherer line))
((char= (aref gatherer (1- length)) #\Space)
(setf gatherer (concatenate 'string gatherer line)))
(t
(setf gatherer
(concatenate 'string gatherer " " line))))))
(t
(setf result (append result (list gatherer) (list line)))
(setf gatherer "")))))
(when gatherer
(setf result (append result (list gatherer))))
result))
(defmethod can-merge-lines-p ((line-1 string) (line-2 string))
(let ((length-1 (length line-1)))
(not (and (> length-1 1)
(char= (aref line-1 (- length-1 1)) #\Space)
(char= (aref line-1 (- length-1 2)) #\Space)))))
(defmethod can-merge-lines-p ((line-1 t) (line-2 t))
(values nil))
(defun handle-setext-headers (document)
"Find headers chunks that can match up with a previous line and make
it so. Also convert line into a header line. Also need to fixup setext
markers that are really horizontal rules markers."
(cond ((= (size (chunks document)) 1)
(when (eq (started-by (first-element (chunks document)))
'line-could-be-header-marker-p)
(setf (started-by (first-element (chunks document)))
'line-is-horizontal-rule-p)))
((>= (size (chunks document)) 2)
(map-window-over-elements
(chunks document) 2 1
(lambda (pair)
(metabang-bind:bind (((p1 p2) pair))
(cond ((and (eq (started-by p2) 'line-could-be-header-marker-p)
(or (empty-p (lines p1))
(eq (ended-by p1) 'line-is-empty-p)))
;; really a horizontal rule)
(setf (started-by p2) 'line-is-horizontal-rule-p))
((and (not (eq (started-by p1) 'line-is-code-p))
(not (empty-p (lines p1)))
(eq (ended-by p1) 'line-could-be-header-marker-p)
(eq (started-by p2) 'line-could-be-header-marker-p))
(make-header p2 (setext-header-markup-class
(first-element (lines p2))))
(setf (first-element (lines p2))
(last-element (lines p1)))
(delete-last (lines p1))
(when (empty-p (lines p1))
(setf (ignore? p1) t)))))))))
(removed-ignored-chunks? document))
(defun removed-ignored-chunks? (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (ignore? chunk) (delete-item (chunks document) chunk))))
document)
(defun setext-header-markup-class (line)
(cond ((char-equal (aref line 0) #\-)
'header2)
((char-equal (aref line 0) #\=)
'header1)
(t
(error "expected a setext header character and got ~A"
(aref line 0)))))
(defun handle-link-reference-titles (document)
"Find title lines that can match up with a link reference line
and make it so. Then parse the links and save them. Finally, remove
those lines."
;; fixup by pulling in titles
(map-window-over-elements
(chunks document) 2 1
(lambda (pair)
(bind (((p1 p2) pair))
(when (and (eq (started-by p1) 'line-is-link-label-p)
(plusp (size (lines p2)))
(line-could-be-link-reference-title-p
(first-element (lines p2))))
(setf (first-element (lines p1))
(concatenate 'string
(first-element (lines p1))
(first-element (lines p2)))
(ended-by p1) 'line-is-link-label-p)
(delete-first (lines p2))
(when (empty-p (lines p2))
(setf (ignore? p2) t))))))
;; parse links
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-is-link-label-p)
(bind (((:values nil link-info)
(scan-to-strings 'link-label
(first-element (lines chunk))))
(id (aref link-info 0))
(url (aref link-info 1))
(title (aref link-info 2))
(properties (aref link-info 3)))
(when title
;; trim off the title delimiters
(setf title (subseq title 1 (- (length title) 1))))
(setf (item-at (link-info document) id)
(make-instance 'link-info
:id id :url url
:title title
:properties properties)
(ignore? chunk) t)))))
;; now remove the unneeded chunks
(removed-ignored-chunks? document)
document)
(defun handle-extended-link-references (document)
;; find them and parse them
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-is-extended-link-label-p)
(bind (((:values nil link-info)
(scan-to-strings 'extended-link-label
(first-element (lines chunk))))
(id (aref link-info 0))
(kind (aref link-info 1))
(contents (aref link-info 2)))
(setf (item-at (link-info document) id)
(make-instance 'extended-link-info
:id id :kind (form-keyword kind) :contents contents)
(ignore? chunk) t)))))
;; now remove the unneeded chunks
(removed-ignored-chunks? document)
document)
(defun line-could-be-link-reference-title-p (line)
"True if the first character is a quote after we skip spaces"
(string-starts-with (strip-whitespace line) "\""))
(defun handle-bullet-lists (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-starts-with-bullet-p)
(push 'bullet (markup-class chunk))
(setf (first-element (lines chunk))
(remove-bullet (first-element (lines chunk))))))))
(defun remove-bullet (line)
;; remove (*|-|+)[.]\s*
; assume is a bullet line
;;?? possibly a hack -- but expedient!
(setf line (strip-whitespace line))
(let ((pos 1)
(length (size line)))
(when (and (>= length 2)
(char-equal (aref line 1) #\.))
(incf pos 1))
(loop while (and (> length pos)
(whitespacep (aref line pos))) do
(incf pos))
(subseq line pos)))
(defun handle-number-lists (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-starts-with-number-p)
(push 'number (markup-class chunk))
(setf (first-element (lines chunk))
(remove-number (first-element (lines chunk))))))))
(defun remove-number (line)
;; remove [0-9]*\.\s*
; assume is a number line
(let ((pos 0)
(length (size line)))
;; digits
(loop while (and (> length pos)
(or (digit-char-p (aref line pos))
(char-is-tab-or-space-p (aref line pos)))) do
(incf pos))
;; required '.'
(incf pos)
;; whitespace
(loop while (and (> length pos)
(whitespacep (aref line pos))) do
(incf pos))
(subseq line pos)))
(defun handle-blockquotes (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-is-blockquote-p)
(push 'quote (markup-class chunk))))))
(defun remove-blockquote (line)
;; removes a single level of blockquoting
(let ((count 0))
;; initial white space
(loop for ch across line
while (whitespacep ch) do (incf count))
;; assume #\>
(incf count)
(subseq line count)))
(defun handle-code (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (eq (started-by chunk) 'line-is-code-p)
(push 'code (markup-class chunk))
(setf (paragraph? chunk) nil)
;; remove last line if it is empty
;;?? is this really the right place to do this...
(when (every #'whitespacep (last-item (lines chunk)))
(delete-last (lines chunk))
)))))
(defun remove-indent (line)
;; removes a single level of indent
(let ((count 0)
(index 0))
;; initial white space
(loop for ch across line
when (char-equal ch #\ ) do (incf count)
when (char-equal ch #\Tab) do (incf count *spaces-per-tab*)
do (incf index)
while (< count *spaces-per-tab*))
(subseq line index)))
;;; canonize-document
(defun canonize-document (document)
(canonize-chunk-markup-class document))
(defun canonize-chunk-markup-class (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(setf (markup-class chunk)
(canonize-markup-class chunk)))))
(defun canonize-markup-class (chunk)
(setf (markup-class chunk)
(sort (markup-class chunk) #'string-lessp)))
(defun cleanup (document)
(remove-empty-bits document)
(handle-paragraph-eval-interactions document)
(unconvert-escapes document)
(iterate-elements
(item-at-1 (properties document) :cleanup-functions)
(lambda (fn)
(funcall fn document))))
(defun remove-empty-bits (document)
(declare (ignorable document))
;; FIXME - if we do this, then the spacing between multiple
;; 'encoded' things (e.g., [link][] `code`) gets lost.
#+(or)
(iterate-elements
(chunks document)
(lambda (chunk)
;;?? yurk -- expediant but ugly
(unless (member 'code (markup-class chunk))
(setf (slot-value chunk 'lines)
(collect-elements
(lines chunk)
:filter
(lambda (line)
(not (and (stringp line)
(zerop
(length (string-trim
+whitespace-characters+ line))))))))))))
(defmethod handle-paragraph-eval-interactions ((document abstract-document))
(iterate-elements (chunks document) #'handle-paragraph-eval-interactions))
(defmethod handle-paragraph-eval-interactions ((chunk chunk))
(unless (chunk-wants-paragraph-p chunk)
#+(or)
(every-element-p
(lines chunk)
(lambda (element)
(and (consp element)
(eq (first element) 'eval)
(null (second (find (second element) *extensions*
:key 'first))))))
(setf (paragraph? chunk) nil)))
(defun chunk-wants-paragraph-p (chunk)
(some-element-p
(lines chunk)
(lambda (line)
(etypecase line
(string (find-if (complement #'whitespacep) line))
(cons (or (not (eq (first line) 'eval))
(null (second (find (second line) *extensions*
:key 'first)))))))))
;;; dead code
#+No
;; this one merges only adjencent pairs and screws that up too b/c it merges ignored things...
(defun merge-chunks-in-document (document)
(map-window-over-elements
(chunks document) 2 1
(lambda (pair)
(metabang-bind:bind (((c1 c2) pair))
(when (can-merge-chunks-p c1 c2)
(merge-chunks c1 c2)))))
(removed-ignored-chunks? document))
cl-markdown-20101006-darcs/dev/reports.lisp 0000644 0001750 0001750 00000030123 11453110206 017042 0 ustar xach xach (in-package #:cl-markdown)
(eval-when (:compile-toplevel :load-toplevel :execute)
(export '(
symbols-exported-with-no-definition
symbols-that-should-be-documented
symbols-documented-by-document
symbols-documented-by-documents
symbols-not-documented-by-multi-document
)))
(defmethod documents ((document document)) (list document))
(defmethod documents ((document multi-document)) (children document))
(defun build-documentation-report (source target document packages
&key (format :html) excluded-symbols
docs-package search-locations)
(cl-markdown:markdown
source
:format format
:stream target
:additional-extensions '(unmentioned-symbols-report
unused-exported-symbols-report
markdown-warnings-report
documented-symbols-report)
:properties `((:documentation-document
. ,document)
(:documentation-documents
. ,(documents document))
(:documentation-packages
. ,(ensure-list packages))
(:documentation-excluded-symbols
. ,excluded-symbols)
,@(when docs-package
`((:docs-package . ,(find-package docs-package))))
(:search-locations
. ,(append
(list
(system-relative-pathname 'cl-markdown "resources/"))
(ensure-list search-locations)))
(:style-sheet . "markdown-report-styles.css"))))
(defun symbols-defined-by-packages (packages &key excluded-symbols)
(sort
(remove-duplicates
(loop for package in (ensure-list packages) append
(let ((package (find-package package)))
(loop for s being the symbols
of package when
(and (eql package (symbol-package s))
(or (fboundp s)
(boundp s)
(find-class s nil)
(get s 'prolog:functor)
(eq (system:variable-information s) :special)
(member s excluded-symbols)
#+allegro
(symbol-is-type-specifier-p s)
;; Could check deftypes if necessary.
))
collect s))))
'string< :key 'symbol-name))
(defun symbols-exported-with-no-definition (packages &key excluded-symbols)
(sort
(remove-duplicates
(loop for package in (ensure-list packages) append
(loop for s being the external-symbols
of package unless
(or (fboundp s)
(boundp s)
(find-class s nil)
(get s 'prolog:functor)
(eq (system:variable-information s) :special)
(member s excluded-symbols)
#+allegro
(symbol-is-type-specifier-p s)
;; Could check deftypes if necessary.
)
collect s)))
'string< :key 'symbol-name))
#+allegro
(defun symbol-is-type-specifier-p (x)
(or (excl:normalize-type
x :loud (lambda (&rest r)
(declare (ignore r))
(return-from symbol-is-type-specifier-p nil)))
t))
(defun symbols-that-should-be-documented
(packages &key
(excluded-packages (list :common-lisp
#+allegro :excl))
excluded-symbols)
(let ((excluded-packages (mapcar #'find-package excluded-packages)))
(sort
(remove-duplicates
(loop for package in (ensure-list packages) append
(loop for s being the external-symbols
of package unless
(or (member s excluded-symbols)
(member (symbol-package s) excluded-packages))
collect s)))
'string< :key 'symbol-name)))
#+(or)
(mapcar #'print
(symbols-that-should-be-documented
(list :db.agraph :db.agraph.parser
:db.agraph.serializer
:sparql
:net.cluster)))
(defun symbols-documented-by-document (document default-package)
(let ((*package* (or (let ((*current-document* document))
(document-property :docs-package))
(find-package default-package)))
(markdown-package (find-package :cl-markdown)))
(flet ((fix-symbol (symbol)
(if (eql (symbol-package symbol) markdown-package)
(intern (symbol-name symbol) *package*)
symbol)))
(mapcar
(lambda (thing)
(etypecase thing
(symbol (fix-symbol thing))
(cons (list (first thing) (fix-symbol (second thing))))))
(cl-containers:collect-keys
(first (cl-containers:item-at-1
(cl-markdown::properties document) :documentation-anchors))
:transform #'car)))))
(defun symbols-documented-by-documents (documents default-package)
(remove-duplicates
(loop for document in documents append
(symbols-documented-by-document document default-package))))
(defun symbols-not-documented-by-multi-document
(multi-doc packages default-package &key
(excluded-packages (list :common-lisp
#+allegro :excl))
(excluded-symbols nil))
(sort
(set-difference
(symbols-that-should-be-documented
packages :excluded-packages excluded-packages
:excluded-symbols excluded-symbols)
(symbols-documented-by-documents multi-doc default-package))
#'string-lessp))
#+ignore
(symbols-not-documented-by-multi-document
(second *last-multi-doc*)
(list :db.agraph :db.agraph.parser
:db.agraph.serializer
:sparql
:net.cluster)
:excluded-symbols (symbols-explicitly-undocumented-for-agraph))
(defextension (unused-exported-symbols-report)
(when (eq phase :render)
(let ((packages (document-property :documentation-packages))
(excluded (document-property :documentation-excluded-symbols))
(*package* (or (document-property :docs-package)
*package*))
(os *output-stream*))
(format os "~&~%")
(format os "~&
Exported Symbols with no apparent use
~%")
(cond (packages
(let ((symbols (symbols-exported-with-no-definition
packages :excluded-symbols excluded)))
(format os "~&
~%")
(format os "~&
From packages:
~%")
(format os "~&~{~&~a~^ ~}"
(mapcar #'package-name packages))
(format os "~&~%")
(cond ((> (length symbols) 0)
(format os "~&
~:d Symbols
~%"
(length symbols))
(loop for s in symbols do
(format os "~&
~s ~%" s)))
(t
(format os "~&All exported symbols are accounted for~%"
(length symbols))))
(when excluded
(format os "~&
~%")
(format os "~&
Ignoring the following ~:d symbols
~%"
(length excluded))
(format os "~&~{~s ~^ ~%~}~%"
excluded)
(format os "~&~%"))))
(t
(format os "~&
There are no packages specified by the
property :documentation-packages~%")))
(format os "~&
~%"))))
(defextension (unmentioned-symbols-report
:arguments ((packages :keyword)
(excluded-symbols :keyword)
(donot-display-excluded-symbols :keyword)))
(when (eq phase :render)
(let ((packages (or packages
(document-property :documentation-packages)))
(excluded (or excluded-symbols
(document-property :documentation-excluded-symbols)))
(documents (document-property :documentation-documents))
(*package* (or (document-property :docs-package)
*package*))
(os *output-stream*))
(format os "~&~%")
(format os "~&
Exported Symbols not mentioned in the documentation
~%")
(cond (packages
(let ((symbols (symbols-not-documented-by-multi-document
documents
packages
*package*
:excluded-symbols excluded)))
(format os "~&
~%")
(format os "~&
From packages:
~%")
(format os "~&~{~&~a ~^ ~}"
(mapcar #'package-name packages))
(format os "~&~%")
(cond ((> (length symbols) 0)
(format os "~&
~:d Undocumented Symbols
~%"
(length symbols))
(loop for s in symbols
do (format os "~&
~s ~%" s)))
(t
(format os "~&All exported symbols are documented.~%")))
(when excluded
(format os "~&
~%")
(format os "~&
Ignoring ~d symbols
~%"
(length excluded))
(unless donot-display-excluded-symbols
(format
os "~&~{~s ~^ ~%~}~%"
excluded))
(format os "~&~%"))))
(t
(format os "~&
There are no packages specified by the
propoerty :documentation-packages~%")))
(format os "~&
~%"))))
;; FIXME -- this is wrong (it's the same as unmentioned right now)
(defextension (undocumented-symbols-report
:arguments ((packages :keyword)
(excluded-symbols :keyword)
(donot-display-excluded-symbols :keyword)))
(when (eq phase :render)
(let ((packages (or packages
(document-property :documentation-packages)))
(excluded (or excluded-symbols
(document-property :documentation-excluded-symbols)))
(documents (document-property :documentation-documents))
(*package* (or (document-property :docs-package)
*package*))
(os *output-stream*))
(format os "~&~%")
(format os "~&
Exported Symbols not mentioned in the documentation
~%")
(cond (packages
(let ((symbols (symbols-not-documented-by-multi-document
documents
packages
*package*
:excluded-symbols excluded)))
(format os "~&
~%")
(format os "~&
From packages:
~%")
(format os "~&~{~&~a ~^ ~}"
(mapcar #'package-name packages))
(format os "~&~%")
(cond ((> (length symbols) 0)
(format os "~&
~:d Undocumented Symbols
~%"
(length symbols))
(loop for s in symbols
do (format os "~&
~s ~%" s)))
(t
(format os "~&All exported symbols are documented.~%")))
(when excluded
(format os "~&
~%")
(format os "~&
Ignoring ~d symbols
~%"
(length excluded))
(unless donot-display-excluded-symbols
(format
os "~&~{~s ~^ ~%~}~%"
excluded))
(format os "~&~%"))))
(t
(format os "~&
There are no packages specified by the
propoerty :documentation-packages~%")))
(format os "~&
~%"))))
(defextension (documented-symbols-report)
(when (eq phase :render)
(let* ((documents (document-property :documentation-documents))
(*package* (or (document-property :docs-package)
*package*))
(os *output-stream*)
(symbols (symbols-documented-by-document
documents *package*)))
(format os "~&~%")
(format os "~&
Documented symbols
~%")
(cond ((> (length symbols) 0)
(format os "~&~:d Documented Symbols
~%"
(length symbols))
(loop for s in symbols
do (format os "~&~s ~%" s)))
(t
(format os "~&All exported symbols are documented.~%")))
(format os "~&~%"))))
(defextension (markdown-warnings-report
:arguments ())
(when (eq phase :render)
(bind ((os *output-stream*)
(document (document-property :documentation-document))
(documents (merge-elements
(warnings document)
(lambda (old new)
(push new old))
(lambda (new)
(list new))
:key 'first :argument 'cdr
:filter (lambda (pair)
(typep (first pair) 'document #+(or) 'child-document))))
(warnings? nil))
(format os "~&~%")
(format os "~&
Markdown Warnings
~%")
(setf documents (sort documents #'string-lessp
:key (compose 'ensure-string 'source 'first)))
(loop for (document warnings) in documents do
(when warnings
(setf warnings? t)
(format os "~&
~%")
(format os "~&
~a
~%"
(short-source (source document)))
(format os "~&
~%")
(loop for warning in (merge-elements
warnings
(lambda (old new)
(declare (ignore new))
(incf (second old))
old)
(lambda (new) (list new 1))
:test 'equal
:return :values) do
(format os "~&- ~a
~%" warning))
(format os "~&
~%")
(format os "~&
~%")))
(unless warnings?
(format os "~%No warnings found.~%"))
(format os "~&
~%"))))
cl-markdown-20101006-darcs/dev/footnotes.lisp 0000644 0001750 0001750 00000011450 11453110206 017366 0 ustar xach xach (in-package #:cl-markdown)
#|
To do:
- allow footnotes to appear on a completely separate page
- do footnotes as a popup window with mouse over
- handle footnotes 'out of band' a la links
Footnotes
{note foo}
{note "This is a note"}
{note "Foo"}
{note This is a note}
(markdown
"That is what he thought.{footnote foo}
[foo]> \"This is a longer note with
linefeeds, *mark-up*, and \\\"escaped\\\" quotes.
I'll be wicked surprised if it works out of the
box.\"
")
Need to
1. get a number
2. add link where the footnote starts
3. add anchor where the footnote starts
4. add footnote text at bottom of document / separate page
5. add link back to anchor in footnote
Our footnote HTML is so heavily influenced by DF that you might think
we just copied it all.
(markdown "
Maybe people{footnote Well, at least one person} find CL-Markdown
to be the bees knees, the cats pajamas and the gnats goulash. In
fact, if computers could dance, you could tell that one had
CL-Markdown installed on it just by watching.{footnote Not really.}
{footnotes}
This was generated {today} at {now}.")
|#
(defclass* footnote-info ()
((id nil ia)
(text nil ia)
(reference-name nil ia)
(name nil ia)))
(eval-when (:load-toplevel :execute)
(setf *extensions* (remove 'footnote *extensions* :key #'first))
(push (list 'footnote t) *extensions*)
(setf *extensions* (remove 'footnotes *extensions* :key #'first))
(push (list 'footnotes t) *extensions*))
;; provides an example of using result during render phase
(defun footnote (phase args result)
;; {documentation text}
(let ((footnotes
(or (document-property :footnote)
(setf (document-property :footnote)
(make-instance 'vector-container)))))
(cond ((eq phase :parse)
(let* ((text (format nil "~{~a ~}" args)))
(when text
(bind ((id (size footnotes))
(fn-basename
(format nil "~d-~a"
id
(format-date "%Y-%m-%d"
(document-property
:date-modified
(get-universal-time)))))
(fn-name (format nil "fn~a" fn-basename))
(ref-name (format nil "fnr~a" fn-basename)))
(insert-item footnotes
(make-instance
'footnote-info
:id id
:name fn-name
:reference-name ref-name
:text text))
(values id)))))
((eq phase :render)
(let ((footnote (item-at footnotes (first result))))
(output-anchor (reference-name footnote))
(format *output-stream*
"~d"
(name footnote)
(1+ (id footnote))))))))
(defun footnotes (phase args result)
(declare (ignore args result))
(ecase phase
(:parse)
(:render
(unless (empty-p (document-property :footnote))
(format *output-stream* "~&")))))
;; not yet
#|
(defun handle-footnote-links (document)
(iterate-elements
(chunks document)
(lambda (chunk)
(when (line-is-footnote-text-p)
(bind (((values nil link-info)
(scan-to-strings '(:sequence footnote-text)
(first-element (lines chunk))))
(id (aref link-info 0))
(text (aref link-info 1)))
(setf (item-at (link-info document) id)
(make-instance 'footnote-text
:id id :title text)
(ignore? chunk) t)))))
;; now remove the unneeded chunks
(removed-ignored-chunks? document)
document)
(defun line-is-footnote-text-p (line)
(scan #.(ppcre:create-scanner '(:sequence footnote-text)) line))
(define-parse-tree-synonym
footnote-label
(:sequence
:start-anchor
(:greedy-repetition 0 3 :whitespace-char-class)
bracketed
#\>
(:greedy-repetition 0 nil :whitespace-char-class)
(:register
(:alternation
(:sequence
#\" (:greedy-repetition 0 nil (:inverted-char-class #\") #\"))
(:greedy-repetition 0 nil :everything)))))
#+(or)
(scan-to-strings
(create-scanner 'footnote-label)
" [a]> why are you here
ok")
#+(or)
(scan-to-strings
(create-scanner 'footnote-label)
" [a]> \"why are you here?
I am here because that is why.
OK? ok!\"")
|# cl-markdown-20101006-darcs/dev/class-defs.lisp 0000644 0001750 0001750 00000011430 11453110206 017370 0 ustar xach xach (in-package #:cl-markdown)
;; someday
(defclass markdown-warning ()
())
(defclass* abstract-document ()
((chunks (make-container 'vector-container) r)
(link-info (make-container 'simple-associative-container
:test #'equalp) r)
(level 0 a)
(markup nil a)
(properties (make-container 'alist-container
:test #'string-equal) r)
(metadata (make-container 'alist-container
:test #'string-equal) r)
(bracket-references (make-container 'flexible-vector-container) r)
(parent nil ir)
(warnings nil a)
(source nil ir)
(destination nil ia) ;more or less the last place
;it was rendered
(children nil ia)))
(defmethod print-object ((object abstract-document) stream)
(print-unreadable-object (object stream :type t :identity t)
(format stream "~a" (short-source (source object)))))
(defclass* document (abstract-document)
())
(defclass* child-document (document)
())
(defclass* multi-document (abstract-document)
())
(defmethod print-object ((object multi-document) stream)
(print-unreadable-object (object stream :type t :identity t)
(format stream "~d children" (length (children object)))))
(defclass* included-document (abstract-document)
())
(defgeneric document-property (name &optional default)
(:documentation "Returns the value of the property `name` of the `*current-document*` or the default if the property is not defined or there is no `*current-document*`."))
(defmethod document-property (name &optional default)
(or (when *current-document*
(multiple-value-bind (value found?)
(item-at-1 (properties *current-document*)
(form-property-name name))
(when found? (return-from document-property (first value)))))
(when (and *current-document*
(parent *current-document*))
(let ((*current-document* (parent *current-document*)))
(document-property name default)))
default))
(defmethod (setf document-property) (value name)
(when *current-document*
(setf (item-at-1 (properties *current-document*)
(form-property-name name))
;; so that we don't lose 'nil'
(list value)))
;;?? weird since nothing happened
(values value))
(defun find-link (id)
(or (item-at-1 (link-info *current-document*) id)
(and (parent *current-document*)
(let ((*current-document* (parent *current-document*)))
(find-link id)))))
(defun form-property-name (name)
(form-keyword (typecase name
(string (intern name
(load-time-value (find-package :keyword))))
(symbol (form-property-name (symbol-name name)))
(t name))))
(defclass* chunk ()
((lines (make-container 'vector-container) r)
(blank-line-before? nil ia)
(blank-line-after? nil ia)
(started-by nil ia)
(ended-by nil ia)
(ignore? nil a)
(markup-class nil ia)
(indentation 0 ia)
(level 0 ia)
(paragraph? nil ia)
(properties (make-container 'alist-container
:test #'string-equal) r)
(stripper? nil ia)))
(defmethod initialize-instance :after ((object chunk) &key lines)
(when lines
(iterate-elements lines (lambda (line) (insert-item (lines object) line)))))
(defmethod print-object ((chunk chunk) stream)
(print-unreadable-object (chunk stream :type t)
(format stream "~a~A/~A ~D lines ~A ~A"
(if (paragraph? chunk) "*" "")
(markup-class chunk)
(level chunk)
(size (lines chunk))
(started-by chunk)
(ended-by chunk))))
(defclass* chunk-parsing-environment ()
((name nil ir)
(chunk-enders nil ia)
(chunk-starters nil ia)
(line-coders nil ia)
(parser-map nil ia)))
(defclass* parsing-environment ()
((chunk-parsing-environment (make-container 'stack-container) r)
(chunk-post-processors nil ia)
(chunk-level 0 ia)
(current-strip "" ia)
(line-code->stripper (make-container 'simple-associative-container
:initial-element nil #+(or) 'null-stripper) r)
(strippers (make-container 'stack-container) r)))
(defun current-chunk-parser ()
(first-item (chunk-parsing-environment *parsing-environment*)))
(defclass* basic-link-info ()
((id nil ia)))
(defclass* link-info (basic-link-info)
((url nil ir)
(title nil ia)
(properties nil ia)))
(defmethod initialize-instance :after ((link link-info) &key properties)
(when (stringp properties)
(setf (properties link)
(collect-window-over-elements
(string->list properties) 2 2
:transform
(lambda (pair) (cons (form-keyword (first pair))
(second pair)))))))
(defclass* extended-link-info (basic-link-info)
((kind nil ir)
(contents nil ia)))
(defmethod print-object ((object link-info) stream)
(print-unreadable-object (object stream :type t :identity t)
(format stream "~A -> ~A" (id object) (url object))))
cl-markdown-20101006-darcs/dev/utilities.lisp 0000644 0001750 0001750 00000033504 11453110206 017365 0 ustar xach xach (in-package #:cl-markdown)
(defun next-block (chunks level)
(let* (;; the first chunk after the current one with a lower level
(pos-level (position-if
(lambda (other)
(< (level other) level))
(rest chunks)))
;; the first chunk after the current one with different markup
;; and level less than or equal to
(pos-markup (position-if
(lambda (other)
(and (<= (level other) level)
(markup-class other) ;paragraphs don't count
(not (samep (markup-class other)
(markup-class (first chunks))))))
(rest chunks)))
pos-style
deeper)
#+Ignore
(format t "~%~d ~2,D:POS: ~A, ~A - ~A"
level (length chunks) pos-level pos-markup (first chunks))
;; remember that there will be another call to rest
(cond #+(or) ((null pos-level)
;; go all the way to the end
(setf deeper (subseq chunks 0)
chunks nil
pos-style :rest))
((or (and pos-level pos-markup (> pos-level pos-markup))
(and pos-level (not pos-markup)))
;(format t " -- level")
(setf deeper (subseq chunks 0 (1+ pos-level))
chunks (nthcdr pos-level chunks)
pos-style :level))
((or (and pos-level pos-markup (> pos-markup pos-level))
(and (not pos-level) pos-markup))
;(format t " -- markup")
(setf deeper (subseq chunks 0 (+ pos-markup 1))
chunks (nthcdr pos-markup chunks)
pos-style :markup))
((and pos-level pos-markup (= pos-level pos-markup))
;(format t " -- level")
(setf deeper (subseq chunks 0 (1+ pos-level))
chunks (nthcdr pos-level chunks)
pos-style :level))
(t
;; nothing found, take the rest
(setf deeper chunks
chunks nil
pos-style :none)))
;(format t "~{~% ~a~}" (collect-elements deeper))
(values deeper chunks pos-style)))
(defmethod render-to-stream (document style stream-specifier)
(with-stream-from-specifier (stream stream-specifier :output
:if-exists :supersede)
(let ((*current-document* document)
(*current-format* style)
(*output-stream* stream))
(setf (level document) 0
(markup document) nil)
(render document style stream))))
(defun ensure-string (it)
(typecase it
(string it)
(symbol (symbol-name it))
(t (format nil "~a" it))))
;; FIXME - bad name
;; this is the parent that is a child-document or document, not, e.g.,
;; an included-document
(defmethod main-parent ((document included-document))
(main-parent (parent document)))
(defmethod main-parent ((document abstract-document))
document)
(defun root-parent (document)
(or (and (parent document)
(root-parent (parent document)))
document))
(defun collect-links (document)
(collect-key-value (link-info document)
:transform (lambda (name link)
(cons name (url link)))
:filter (lambda (k v)
(declare (ignore k))
(typep v 'link-info))))
(defun starts-with (string prefix)
(let ((mismatch (mismatch prefix string)))
(or (not mismatch) (= mismatch (length prefix)))))
(defun markdown-warning (msg &rest args)
(let* ((*print-readably* nil)
(warning (apply #'format nil msg args)))
(when *current-document*
(push (cons (main-parent *current-document*) warning)
(warnings (root-parent *current-document*))))
(fresh-line *debug-io*)
(write-string warning *debug-io*)
(terpri *debug-io*)))
(eval-always
(defun _mark-range (array start end)
(loop for a from (char-code start) to (char-code end) do
(setf (sbit array a) 1)))
(defun _mark-one (array ch)
(setf (sbit array (char-code ch)) 1)))
(defparameter +first-name-characters+
(let ((array (make-array 255 :element-type 'bit :initial-element 0)))
(_mark-range array #\a #\z)
(_mark-range array #\A #\Z)
array))
(defparameter +name-characters+
(let ((array (copy-seq +first-name-characters+)))
(_mark-range array #\0 #\9)
(_mark-one array #\_)
(_mark-one array #\-)
(_mark-one array #\.)
(_mark-one array #\:)
array))
(defun html-safe-name (name)
;; Copied from HTML-Encode
;;?? this is very consy
;;?? crappy name
(declare (type simple-string name))
(let ((output (make-array (truncate (length name) 2/3)
:element-type 'character
:adjustable t
:fill-pointer 0))
(first? t))
(with-output-to-string (out output)
(loop for char across name
for code = (char-code char)
for valid = +first-name-characters+ then +name-characters+
do (cond ((and (< code 255)
(= (sbit valid code) 1))
(write-char char out))
(t
;; See http://www.w3.org/TR/html4/types.html#h-6.2
;; ID and NAME tokens must begin with a letter ([A-Za-z])
;; and may be followed by any number of letters,
;; digits ([0-9]), hyphens ("-"), underscores ("_"),
;; colons (":"), and periods (".").
(when first?
(write-char #\x out))
(format out ":~:@(~16,r~)" code)))
(setf first? nil)))
(coerce output 'simple-string)))
(defun string->list (string &key (stop-word-p (constantly nil))
(ignore-character-p (constantly nil)))
(let ((word-array (make-array (length string) :element-type 'character))
(results nil)
(index 0)
(in-quote-p nil))
(labels ((grab-char (ch)
(setf (aref word-array index) ch)
(incf index))
(add-word (word)
(push word results))
(maybe-add-word ()
(let ((word (coerce (subseq word-array 0 index) 'string)))
(unless (funcall stop-word-p word)
(add-word (strip-whitespace word))))
(setf index 0)))
(loop for i below (length string)
for ch = (aref string i) do
(cond ((char= ch #\\)
(grab-char ch)
(incf i)
(grab-char (aref string i)))
((char= ch #\")
(setf in-quote-p (not in-quote-p))
#+(or)
(grab-char #\"))
((and (not in-quote-p) (> index 0) (whitespacep ch))
;; have a word
(maybe-add-word))
((and (not in-quote-p) (funcall ignore-character-p ch))
)
(t
(grab-char ch))))
(when (> index 0)
(maybe-add-word))
(nreverse results))))
(defun process-brackets (document current-line line-iterator)
(bind ((output (make-array 4096
:element-type 'character
:adjustable t
:fill-pointer 0))
(depth 0)
(location 0)
(buffers (bracket-references document))
(buffer-count (size buffers))
(current-buffer nil))
(with-output-to-string (out output)
(block
iteration-block
(flet ((write-buffer-count (count)
(format out " ~d" count)
(setf current-buffer nil))
(add-char (ch)
(if current-buffer
(vector-push-extend ch current-buffer)
(write-char ch out)))
(start-buffer ()
(setf current-buffer
(make-array 4096
:element-type 'character
:adjustable t
:fill-pointer 0))))
(flet ((process-brackets-in-line (line)
;;(print (list :line line))
(with-iterator (iterator line
:treat-contents-as :characters
:skip-empty-chunks? nil)
(iterate-elements
iterator
(lambda (ch)
;;(print (list ch (not (null current-buffer))))
(incf location)
(cond ((char= ch #\\)
(unless (move-forward-p iterator)
(error "Invalid escape at char ~d" location))
(add-char ch)
(add-char (next-element iterator)))
((and (= depth 1) (not current-buffer)
(whitespacep ch))
;; finished reading the command name
;; (don't write the ws)
(start-buffer))
((char= ch #\{)
(incf depth)
(add-char ch))
((char= ch #\})
(decf depth)
(cond ((= depth 0)
(insert-item
buffers
(coerce current-buffer 'simple-string))
(write-buffer-count buffer-count))
((< depth 0)
;; FIXME -- an error
(setf depth 0)))
(add-char ch))
(t
(add-char ch))))))
;; if no brackets to process at the end of a line, bail
(when (= depth 0)
(return-from iteration-block nil))
(if (and (= depth 1) (not current-buffer))
(start-buffer)
(add-char #\Newline))))
(process-brackets-in-line current-line)
(move-forward line-iterator)
(iterate-elements
line-iterator
(lambda (line)
(process-brackets-in-line line))))))
(coerce output 'simple-string))))
#+ignore
(let ((li (make-iterator "a b
c d e
f" :treat-contents-as :lines)))
(iterate-elements
li
(lambda (x)
(iterate-elements li
(lambda (y)
(let ((ci (make-iterator y :treat-contents-as :characters :skip-empty-chunks? nil)))
(iterate-elements ci #'print)))))))
(defun short-source (source)
(typecase source
(pathname source)
(string
(format nil "~a~@[...~]"
(substitute-if #\Space
(lambda (ch)
(or (char= ch #\newline)
(char= ch #\linefeed)))
(subseq source 0 (min 50 (length source))))
(> (length source) 50)))
(t (format nil "Something of type ~s"
(type-of source)))))
(defun could-be-html-tag-p (string start)
;; assumes that start == the index _after_ the #\<
(let ((state :start))
(loop for index from start below (length string)
for ch = (schar string index) do
(ecase state
(:start (when (whitespacep ch) (return nil))
(setf state :running))
(:running
(case ch
(#\' (setf state :single-quote))
(#\" (setf state :double-quote))
(#\> (return index))))
(:single-quote
(case ch
(#\' (setf state :running))))
(:double-quote
(case ch
(#\" (setf state :running))))))))
;;
;; ^
(defun stream-string-for-html (string stream)
(declare (simple-string string))
(let ((next-index 1)
(last-index nil))
(with-output (out stream)
(loop for char across string
do (cond
((char= char #\&) (write-string "&" out))
((char= char #\<)
(setf last-index (could-be-html-tag-p string next-index))
(if last-index
(write-char char out)
(write-string "<" out)))
((and (null last-index) (char= char #\>))
(write-string ">" out))
(t (write-char char out)))
(when (and last-index (> next-index last-index))
(setf last-index nil))
(incf next-index)))))
#+(or)
(stream-string-for-html "hello a > b and b < a " nil)
(defun encode-string-for-html (string)
(stream-string-for-html string nil))
;; Copied from HTML-Encode
;;?? this is very consy
;;?? crappy name
(defun encode-pre (string)
(declare (simple-string string))
(let ((output (make-array (truncate (length string) 2/3)
:element-type 'character
:adjustable t
:fill-pointer 0)))
(with-output-to-string (out output)
(loop for char across string
do (case char
((#\&) (write-string "&" out))
((#\<) (write-string "<" out))
((#\>) (write-string ">" out))
(t (write-char char out)))))
(coerce output 'simple-string)))
;; Copied from HTML-Encode
;;?? this is very consy
;;?? crappy name
;;?? this is really bugging me -- what gross code and it's repeated FOUR times
;;
(defun encode-string-for-title (string)
(declare (simple-string string))
(if (find #\' string :test #'char=)
(let ((output (make-array (truncate (length string) 2/3)
:element-type 'character
:adjustable t
:fill-pointer 0)))
(with-output-to-string (out output)
(loop for char across string
do (case char
;; (code-char 39) ==> #\'
((#\') (write-string "'" out))
(t (write-char char out)))))
(coerce output 'simple-string))
string))
(defun find-include-file (pathname)
(bind ((pathname (ensure-string pathname))
(search-locations (ensure-list (document-property :search-locations)))
(result
(or (probe-file (merge-pathnames pathname))
;; look in search-locations
(some (lambda (location)
(probe-file (merge-pathnames pathname location)))
search-locations))))
(unless result
(markdown-warning
"Unable to find ~a in any of the search-locations ~{~a~^, ~}"
pathname search-locations))
result))
(defun process-child-markdown (text phase &key (transfer-data nil))
(bind (((:values child output)
(markdown text
:parent *current-document*
:format (if (eq phase :parse) :none *current-format*)
:properties '((:omit-initial-paragraph t)
(:omit-final-paragraph t))
:stream nil
:document-class 'included-document)))
(push child (children *current-document*))
(when transfer-data
(transfer-link-info *current-document* child "")
(transfer-document-metadata *current-document* child)
(transfer-selected-properties
*current-document* child
(set-difference (collect-keys (properties child))
(list :omit-initial-paragraph :omit-final-paragraph))))
(ecase phase
(:parse child)
(:render (strip-whitespace output)))))
(defun asdf-system-source-file (system-name)
(let ((system (asdf:find-system system-name)))
(make-pathname
:type "asd"
:name (asdf:component-name system)
:defaults (asdf:component-relative-pathname system))))
(defun asdf-system-source-directory (system-name)
(make-pathname :name nil
:type nil
:defaults (asdf-system-source-file system-name)))
(defun system-relative-pathname (system pathname &key name type)
(relative-pathname (asdf-system-source-directory system)
pathname :name name :type type))
cl-markdown-20101006-darcs/dev/dead-code/ 0000755 0001750 0001750 00000000000 11453110206 016261 5 ustar xach xach cl-markdown-20101006-darcs/dev/dead-code/lml2.lisp 0000644 0001750 0001750 00000015455 11453110206 020032 0 ustar xach xach (in-package #:cl-markdown)
(defparameter *markup->lml2*
(make-container
'simple-associative-container
:test #'equal
:initial-contents
'((header1) (nil :h1)
(header2) (nil :h2)
(header3) (nil :h3)
(header4) (nil :h4)
(header5) (nil :h5)
(header6) (nil :h6)
(bullet) ((:ul) :li)
(code) ((:pre :code) nil)
(number) ((:ol) :li)
(quote) ((:blockquote) nil)
(horizontal-rule) (nil :hr))))
;;; ---------------------------------------------------------------------------
(defmethod render ((document document) (style (eql :lml2)) stream)
(let ((*current-document* document))
(setf (level document) 0
(markup document) nil)
(let* ((chunks (collect-elements (chunks document)))
(result (lml2-list->tree chunks)))
(if stream
(format stream "~S" result)
result))))
;;; ---------------------------------------------------------------------------
(defun lml2-marker (chunk)
(bind ((markup (markup-class-for-lml2 chunk)))
(first markup)))
;;; ---------------------------------------------------------------------------
(defmethod render-to-lml2 ((chunk chunk))
(bind ((block (collect-elements
(lines chunk)
:transform (lambda (line)
(render-to-lml2 line))))
(markup (second (markup-class-for-lml2 chunk)))
(paragraph? (paragraph? chunk)))
(cond ((and paragraph? markup)
(values `(,markup (:P ,@block)) t))
(paragraph?
(values `(:P ,@block) t))
(markup
(values `(,markup ,@block) t))
(t
(values block nil)))))
;;; ---------------------------------------------------------------------------
(defmethod markup-class-for-lml2 ((chunk chunk))
(when (markup-class chunk)
(let ((translation (item-at-1 *markup->lml2* (markup-class chunk))))
(unless translation
(warn "No translation for '~A'" (markup-class chunk)))
translation)))
;;; ---------------------------------------------------------------------------
(defmethod render-to-lml2 ((chunk list))
(render-span-to-lml2 (first chunk) (rest chunk)))
;;; ---------------------------------------------------------------------------
(defmethod render-to-lml2 ((chunk string))
;;?? unlovely
(format nil "~A" chunk))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'strong)) body)
`(:strong ,@body))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'mail)) body)
(let ((address (first body)))
`((:a :href ,(format nil "mailto:~A" address)) ,address)))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'emphasis)) body)
`(:em ,@body))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'strong-em)) body)
`(:strong (:em ,@body)))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'code)) body)
`(:code ,(render-to-lml2 (first body))))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'entity)) body)
(first body))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'reference-link)) body)
(bind (((text &optional (id text)) body)
(link-info (item-at-1 (link-info *current-document*) id)))
(if link-info
`((:a :href ,(url link-info) ,@(awhen (title link-info) `(:title ,it)))
,text)
`,text)))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'inline-link)) body)
(bind (((text &optional (url "") title) body))
`((:a :href ,url ,@(awhen title `(:title ,it)))
,text)))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'link)) body)
(bind ((url body))
`((:a :href ,@url) ,@url)))
;;; ---------------------------------------------------------------------------
(defmethod render-span-to-lml2 ((code (eql 'html)) body)
(html-encode:encode-for-pre (first body)))
(defun lml2-list->tree (chunks &key (level nil))
(unless level
(setf level (or (and (first chunks) (level (first chunks))) 0)))
(labels ((do-it (chunks level)
;;?? rather inpenetrable... don't understand at the level I should...
(apply-mark
(lml2-marker (first chunks))
(let (output append? result)
(loop for rest = chunks then (rest rest)
for chunk = (first rest) then (first rest)
while chunk
for new-level = (level chunk)
do (setf (values output append?) (render-to-lml2 chunk))
do (format t "~%C(~D/~D): ~A, ~A" level new-level append? chunk)
when (and (= level new-level) append?) do
(setf result `(,output ,@result))
when (and (= level new-level) (not append?)) do
(setf result `(,@output ,@result))
when (< level new-level) do
(multiple-value-bind (block remaining method)
(next-block rest new-level)
(let ((inner (do-it block (1+ level))))
; (format t "~%--- ~A" method)
(setf rest remaining)
(ecase method
(:level (if (listp (first result))
(push-end inner (first result))
(push inner result)))
(:markup (push inner result))
(:none
(setf result `(,inner ,@result))))))
when (> level new-level) do
(warn "unexpected chunk level"))
(reverse result)))))
(apply #'do-it chunks level)))
;;; ---------------------------------------------------------------------------
(defun apply-mark (mark rest)
(cond ((null mark) rest)
((consp mark)
(if (length-1-list-p mark)
`(,(first mark) ,@(apply-mark (rest mark) rest))
`(,(first mark) (,@(apply-mark (rest mark) rest)))))
(t
(error "unhandled case"))))
cl-markdown-20101006-darcs/dev/macros.lisp 0000644 0001750 0001750 00000006540 11453110206 016636 0 ustar xach xach (in-package #:cl-markdown)
(defmacro defsimple-extension (name &body body)
"Create an extension (a function named `name`) with no arguments that
does not depend on the markdown phase and which does not use the result.
These are handy for simple text substitutions."
(with-gensyms (phase arguments result)
`(progn
(pushnew (list ',name t) *extensions* :key #'car)
(defun ,name (,phase ,arguments ,result)
(declare (ignore ,phase ,arguments ,result))
,@body)
,@(%import/export-symbol name))))
(defun %validate-defextension-arguments (arguments)
(loop for argument in (ensure-list arguments) do
(cond ((atom argument)
(when (eq (symbol-package argument) #.(find-package :keyword))
(error "Argument names may not be keywords and ~s is not"
argument)))
(t
(unless (every (lambda (facet)
(member facet '(:required :keyword :whole)))
(rest argument))
(error "Invalid argument facets in ~s" (rest argument)))))))
(defun %collect-arguments (arguments kind)
(loop for argument in (ensure-list arguments)
when (and (consp argument)
(member kind (rest argument))) collect
(first argument)))
(defun %collect-positionals (arguments)
(loop for argument in (ensure-list arguments)
when (or (atom argument)
(and (consp argument)
(not (member :keyword (rest argument))))) collect
(first (ensure-list argument))))
(defparameter *extensions* nil)
(defmacro defextension ((name &key arguments (insertp nil) (exportp t))
&body body)
(%validate-defextension-arguments arguments)
(bind ((keywords (%collect-arguments arguments :keyword))
(requires (%collect-arguments arguments :required))
(whole (%collect-arguments arguments :whole))
(positionals (%collect-positionals arguments)))
(assert (<= (length whole) 1)
nil "At most one :whole argument is allowed.")
(assert (null (intersection whole keywords))
nil "Keyword arguments cannot be wholes")
`(progn
(setf *extensions* (remove ',name *extensions* :key #'first))
(push (list ',name ,insertp) *extensions*)
(defun ,name (phase args result)
(declare (ignorable phase args result))
(bind (,@(loop for positional in positionals
unless (member positional whole) collect
`(,positional (pop args)))
,@(loop for keyword in keywords collect
`(,keyword
(getf args ,(intern (symbol-name keyword) :keyword)
nil)))
,@(when whole
`((,(first whole)
;; remove keywords from args
(progn
,@(loop for keyword in keywords collect
`(,keyword
(remf args
,(intern (symbol-name keyword) :keyword))))
(if (length-1-list-p args) (first args) args))))))
,@(loop for require in requires collect
`(assert ,require nil ,(format nil "~s is required" require)))
,@body
,@(unless insertp nil)))
,@(when exportp
(%import/export-symbol name)))))
(defun %import/export-symbol (name)
`((eval-when (:compile-toplevel :load-toplevel :execute)
(import ',name ,(find-package :cl-markdown-user))
(export ',name ,(find-package :cl-markdown-user)))))
(defmacro aand+ (&rest args)
"Anaphoric nested AND.
Binds the symbol `it' to the value of the preceding `arg.'"
(cond ((null args) t)
((null (cdr args)) (car args))
(t `(aif ,(car args) (aand ,@(cdr args))))))
cl-markdown-20101006-darcs/dev/definitions.lisp 0000644 0001750 0001750 00000002000 11453110206 017650 0 ustar xach xach (in-package #:cl-markdown)
(defparameter *spaces-per-tab* 4)
(defparameter *parsing-environment* nil)
(defparameter *chunk-parsing-environments*
(make-container 'simple-associative-container))
(defparameter *spanner-parsing-environments*
(make-container 'simple-associative-container :test #'equal))
(defparameter *horizontal-rule-count-threshold* 3)
(defparameter *default-stream* *standard-output*)
(defparameter *default-format* :html)
(defvar *output-stream* nil)
(defvar *current-indentation-level* 0)
(defparameter *current-document* nil)
(defparameter *current-format* nil)
(defparameter *render-active-functions*
'(table-of-contents property set-property anchor footnote footnotes
today now include include-if comment))
(defparameter *parse-active-functions*
'(table-of-contents property set-property anchor footnote footnotes
include include-if comment))
(defparameter *block-level-html-tags*
'(address blockquote div fieldset
h1 h2 h3 h4 h5 h6
hr legend p pre ul ol li dl dd))
cl-markdown-20101006-darcs/dev/html.lisp 0000644 0001750 0001750 00000041312 11453110206 016312 0 ustar xach xach (in-package #:cl-markdown)
(defstruct (html-markup
(:conc-name markup-)
(:print-object print-html-markup))
name outer inner tag encoding-method contentlessp (nestsp t))
(defmethod print-html-markup (markup stream)
(print-unreadable-object (markup stream :type nil :identity t)
(bind (((:struct markup- name inner outer encoding-method) markup))
(format stream "~a : ~a ~a ~a"
name outer inner encoding-method))))
(defparameter *markup->html*
(make-container
'simple-associative-container
:test #'equal
:initial-contents
(loop for datum in '((header1 nil nil "h1")
(header2 nil nil "h2")
(header3 nil nil "h3")
(header4 nil nil "h4")
(header5 nil nil "h5")
(header6 nil nil "h6")
(bullet ("ul") ("li") nil :nestsp t)
(code ("pre" "code") nil nil :new-method encode-pre)
(number ("ol") ("li") nil :nestsp nil)
(quote ("blockquote") nil nil)
(horizontal-rule nil nil "hr" :contentlessp t)) nconc
(bind (((tag outer inner markup &key
new-method
contentlessp (nestsp t))
datum))
(list (list tag)
(make-html-markup
:name tag
:outer outer
:inner inner
:tag markup
:encoding-method new-method
:contentlessp contentlessp
:nestsp nestsp))))))
(defvar *magic-space-p* nil)
(defvar *magic-line-p* 0)
(defparameter *magic-space* #\Space)
(defparameter *magic-line* nil)
#+(or)
(setf *magic-space* #\| *magic-line* #\~)
(defgeneric render-to-html (stuff encoding-method)
(:documentation ""))
(defmethod render ((document abstract-document) (style (eql :html)) stream)
(declare (ignore stream))
(setf *magic-space-p* nil
*magic-line-p* 0)
(render-to-html document nil))
(defun html-block-markup (chunk)
(aand+
(markup-class-for-html chunk)
(markup-outer it)))
(defun html-inner-block-markup (chunk)
(aand
(markup-class-for-html chunk)
(markup-inner it)))
(defmethod render-to-html ((chunk chunk) encoding-method)
(bind (((:struct markup- (markup tag)
(new-method encoding-method) contentlessp)
(markup-class-for-html chunk))
(paragraph? (paragraph? chunk)))
(cond (contentlessp
(format *output-stream* "<~a/>" markup))
(t
(encode-html chunk (or new-method encoding-method)
markup (when paragraph? "p"))))))
;;?? same code as below
(defmethod encode-html ((stuff chunk) encoding-method &rest codes)
(declare (dynamic-extent codes))
(setf *magic-line-p* 0)
(cond ((null codes)
(let ((code-p (member 'code (markup-class stuff))))
(iterate-elements
(lines stuff)
(lambda (line)
(render-to-html line encoding-method)
(when code-p (incf *magic-line-p*))))
;(fresh-line *output-stream*)
))
((null (first codes))
(apply #'encode-html stuff encoding-method (rest codes)))
(t (format *output-stream* "<~a>" (first codes))
(apply #'encode-html stuff encoding-method (rest codes))
(format *output-stream* "~a>" (first codes)))))
;;?? same code as above
(defmethod encode-html ((stuff list) encoding-method &rest codes)
(declare (dynamic-extent codes))
(cond ((null codes)
(iterate-elements
stuff
(lambda (line)
(render-to-html line encoding-method))))
((null (first codes))
(apply #'encode-html stuff encoding-method (rest codes)))
(t (format *output-stream* "<~A>" (first codes))
(apply #'encode-html stuff encoding-method (rest codes))
(format *output-stream* "~A>" (first codes)))))
(defmethod markup-class-for-html ((chunk chunk))
(if (markup-class chunk)
(let ((translation (item-at-1 *markup->html* (markup-class chunk))))
(unless translation
(markdown-warning "No translation for markup class '~A'"
(markup-class chunk)))
translation)
(load-time-value (make-html-markup))))
(defmethod render-to-html ((chunk list) encoding-method)
(render-span-to-html (first chunk) (rest chunk) encoding-method))
(defmethod render-to-html ((line string) encoding-method)
(when *magic-space-p*
(setf *magic-space-p* nil)
(princ *magic-space* *output-stream*))
(when (> *magic-line-p* 0)
(when *magic-line*
(princ *magic-line* *output-stream*))
(terpri *output-stream*))
(format *output-stream* "~a"
(funcall (or encoding-method 'encode-string-for-html) line))
(setf *magic-space-p* t))
(defun output-html (string &rest codes)
(declare (dynamic-extent codes))
(cond ((null codes) (princ (first string) *output-stream*))
(t (format *output-stream* "<~(~a~)>" (first codes))
(apply #'output-html string (rest codes))
(format *output-stream* "~(~a~)>" (first codes)))))
(defmethod render-span-to-html ((code (eql 'strong)) body encoding-method)
(declare (ignore encoding-method))
(output-html body 'strong)
(setf *magic-space-p* nil))
(defmethod render-span-to-html ((code (eql 'mail)) body encoding-method)
(declare (ignore encoding-method))
(let ((address (first body)))
(output-link (format nil "mailto:~A" address) nil address)))
(defmethod render-span-to-html ((code (eql 'emphasis)) body encoding-method)
(declare (ignore encoding-method))
(output-html body 'em)
(setf *magic-space-p* nil))
(defmethod render-span-to-html ((code (eql 'strong-em)) body encoding-method)
(declare (ignore encoding-method))
(output-html body 'strong 'em)
(setf *magic-space-p* nil))
(defmethod render-span-to-html
((code (eql 'escaped-character)) body encoding-method)
(declare (ignore encoding-method))
(let ((char (aref (first body) 0)))
(cond ((char= #\< char) (princ "<" *output-stream*))
((char= #\> char) (princ ">" *output-stream*))
(t (output-html body))))
(setf *magic-space-p* nil))
(defmethod render-span-to-html ((code (eql 'code)) body encoding-method)
(declare (ignore encoding-method))
(format *output-stream* "")
(setf *magic-space-p* nil)
(dolist (bit body)
(render-to-html bit 'encode-pre))
(format *output-stream* "
")
(setf *magic-space-p* nil))
(defmethod render-span-to-html ((code (eql 'entity)) body encoding-method)
(declare (ignore encoding-method))
(setf *magic-space-p* nil
*magic-line-p* -1)
(output-html body)
(setf *magic-space-p* nil
*magic-line-p* -1))
(defmethod render-span-to-html
((code (eql 'reference-link)) body encoding-method)
(declare (ignore encoding-method))
(bind (((:values text id nil)
(if (length-1-list-p body)
(values (first body) (first body) nil)
(values (butlast body 1) (first (last body)) t)))
(link-info (find-link id)))
(cond ((not (null link-info))
;; it _was_ a valid ID
(generate-link-output link-info text))
(t
(markdown-warning "No reference found for link ~s" id)
(format *output-stream* "~a" (if (consp text) (first text) text))
(setf *magic-space-p* nil)))))
(defmethod generate-link-output ((link-info link-info) text)
(output-link (url link-info) (title link-info) text (properties link-info)))
(defmethod generate-link-output ((link-info extended-link-info) text)
;; you didn't really want it to be fast did you...?
(generate-link-output-for-kind (kind link-info) link-info text))
(defmethod render-span-to-html
((code (eql 'inline-link)) body encoding-method)
(declare (ignore encoding-method))
(bind (((text &optional (url "") title) body))
(output-link url title (list text))))
(defmethod render-span-to-html ((code (eql 'link)) body encoding-method)
(declare (ignore encoding-method))
(let ((link (first body)))
(output-link link nil link)))
(defmethod render-span-to-html
((code (eql 'reference-image)) body encoding-method)
(declare (ignore encoding-method))
(bind (((:values text id nil)
(if (length-1-list-p body)
(values (first body) (first body) nil)
(values (butlast body 1) (first (last body)) t)))
(link-info (find-link id)))
(cond ((not (null link-info))
;; it _was_ a valid ID
(output-image (url link-info) (title link-info) text
(properties link-info)))
(t
;;?? hackish
(markdown-warning "No reference found for image link ~s" id)
(format *output-stream* "~a (image)"
(if (consp text) (first text) text))
(setf *magic-space-p* nil)))))
(defmethod render-span-to-html
((code (eql 'inline-image)) body encoding-method)
(declare (ignore encoding-method))
(bind (((text &optional (url "") title) body))
(output-image url title text)))
(defun output-link (url title text &optional properties)
(cond ((not (null url))
(format *output-stream* "" *output-stream*)
(setf *magic-space-p* nil)
(encode-html (ensure-list text) nil)
(format *output-stream* "")
(setf *magic-space-p* nil))
(t
)))
(defun output-image (url title text &optional properties)
(cond ((not (null url))
(format *output-stream*
"
" *output-stream*)
(setf *magic-space-p* nil))
(t
)))
(defmethod render-span-to-html ((code (eql 'html)) body encoding-method)
;; hack!
(let ((output (first body)))
(etypecase output
(string
(output-html (list output #+(or) (encode-pre output))))
(list
(render-span-to-html (first output) (rest output) encoding-method)))))
;; Special cases R us.
(defmethod render-span-to-html ((code (eql 'break)) body encoding-method)
(encode-html body encoding-method)
(format *output-stream* "
~%"))
(defun stream-out-markup (markup reverse)
(dolist (marker (if reverse (reverse markup) markup))
(let ((cr? (member marker
*block-level-html-tags* :test 'string=)))
(when (and (not reverse) cr?)
(terpri *output-stream*))
(format *output-stream* "<~a~a>"
(if reverse "/" "") marker))))
;; FIXME - in case you're wondering, this is an ugly bit of code
(defmethod render-to-html ((document abstract-document) encoding-method)
(bind ((current-chunk nil)
(wrap-in-html (add-html-header-p document)))
(labels ((render-block (block level markup inner?)
; (print (list :rb level inner? (first block)))
(setf *magic-space-p* nil)
(let ((add-markup? (not (eq (first block) current-chunk)))
(real-markup
(if (and (not inner?) (length-1-list-p block))
(append
markup (html-inner-block-markup (first block)))
markup)))
(when add-markup?
(stream-out-markup real-markup nil))
(cond ((or (length-1-list-p block)
)
(render-to-html (first block) encoding-method))
((not add-markup?)
(render-to-html (first block) encoding-method)
(do-it (rest block) level))
(t
(setf current-chunk (and inner? (first block)))
(do-it block level)))
(when add-markup?
(stream-out-markup real-markup t))))
(do-it (chunks level)
; (print (list :di level (first chunks)))
(loop for rest = chunks then (rest rest)
for chunk = (first rest) then (first rest)
while chunk
for new-level = (level chunk)
when (= level new-level) do
(let ((index (inner-block rest))
(inner-markup (html-inner-block-markup chunk)))
(render-block (subseq rest 0 index)
level inner-markup t)
(setf rest (nthcdr (1- index) rest)))
when (< level new-level) do
(multiple-value-bind (block remaining method)
(next-block rest new-level)
(declare (ignore method))
(render-block
block new-level (html-block-markup chunk) nil)
(setf rest remaining)))))
(when wrap-in-html
(generate-html-header))
(do-it (collect-elements (chunks document)) (level document))
(when wrap-in-html
(format *output-stream* "~&