pax_global_header 0000666 0000000 0000000 00000000064 14624642056 0014523 g ustar 00root root 0000000 0000000 52 comment=8672fb3dd411dbea45bfafa060013fcbfdccd045
golang-github-johanneskaufmann-html-to-markdown-1.6.0/ 0000775 0000000 0000000 00000000000 14624642056 0023004 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/ 0000775 0000000 0000000 00000000000 14624642056 0024344 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/ISSUE_TEMPLATE/ 0000775 0000000 0000000 00000000000 14624642056 0026527 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/ISSUE_TEMPLATE/bug_report.md 0000664 0000000 0000000 00000001020 14624642056 0031212 0 ustar 00root root 0000000 0000000 ---
name: Bug report
about: Create a report to help us improve
title: "\U0001F41B Bug"
labels: bug
assignees: ''
---
**Describe the bug**
A clear and concise description of what the bug is.
**HTML Input**
```html
Title
```
**Generated Markdown**
````markdown
# Title
````
**Expected Markdown**
````markdown
# Title!!!
````
**Additional context**
Add any other context about the problem here. For example, if you changed the default options or used a plugin. Also adding the version from the `go.mod` is helpful.
golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/dependabot.yml 0000664 0000000 0000000 00000000425 14624642056 0027175 0 ustar 00root root 0000000 0000000 # Please see the documentation for all configuration options:
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
version: 2
updates:
- package-ecosystem: "gomod"
directory: "/"
schedule:
interval: "weekly"
golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/workflows/ 0000775 0000000 0000000 00000000000 14624642056 0026401 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/.github/workflows/go.yml 0000664 0000000 0000000 00000001572 14624642056 0027536 0 ustar 00root root 0000000 0000000 name: Go
on:
push:
branches: [ master ]
pull_request:
branches: [ master ]
jobs:
build:
name: Build
runs-on: ubuntu-latest
steps:
- name: Set up Go 1.x
uses: actions/setup-go@v2
with:
go-version: ^1.13
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Get dependencies
run: |
go get -v -t -d ./...
if [ -f Gopkg.toml ]; then
curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
dep ensure
fi
- name: Build
run: go build -v .
- name: Test
run: go test -v -race -coverprofile=coverage.txt -covermode=atomic .
- name: Upload Coverage report to CodeCov
uses: codecov/codecov-action@v2
with:
token: ${{secrets.CODECOV_TOKEN}}
file: ./coverage.txt
golang-github-johanneskaufmann-html-to-markdown-1.6.0/.gitignore 0000664 0000000 0000000 00000000313 14624642056 0024771 0 ustar 00root root 0000000 0000000 # Binaries for programs and plugins
*.exe
*.exe~
*.dll
*.so
*.dylib
# Test binary, build with `go test -c`
*.test
# Output of the go coverage tool, specifically when used with LiteIDE
*.out
.DS_Store
golang-github-johanneskaufmann-html-to-markdown-1.6.0/CONTRIBUTING.md 0000664 0000000 0000000 00000000000 14624642056 0025223 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/LICENSE 0000664 0000000 0000000 00000002062 14624642056 0024011 0 ustar 00root root 0000000 0000000 MIT License
Copyright (c) 2018 Johannes Kaufmann
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
golang-github-johanneskaufmann-html-to-markdown-1.6.0/README.md 0000664 0000000 0000000 00000024070 14624642056 0024266 0 ustar 00root root 0000000 0000000 # html-to-markdown
[](https://goreportcard.com/report/github.com/JohannesKaufmann/html-to-markdown)
[](https://codecov.io/gh/JohannesKaufmann/html-to-markdown)
[](http://godoc.org/github.com/JohannesKaufmann/html-to-markdown)
Convert HTML into Markdown with Go. It is using an [HTML Parser](https://github.com/PuerkitoBio/goquery) to avoid the use of `regexp` as much as possible. That should prevent some [weird cases](https://stackoverflow.com/a/1732454) and allows it to be used for cases where the input is totally unknown.
## Installation
```
go get github.com/JohannesKaufmann/html-to-markdown
```
## Usage
```go
import (
"fmt"
"log"
md "github.com/JohannesKaufmann/html-to-markdown"
)
converter := md.NewConverter("", true, nil)
html := `Important`
markdown, err := converter.ConvertString(html)
if err != nil {
log.Fatal(err)
}
fmt.Println("md ->", markdown)
```
If you are already using [goquery](https://github.com/PuerkitoBio/goquery) you can pass a selection to `Convert`.
```go
markdown, err := converter.Convert(selec)
```
### Using it on the command line
If you want to make use of `html-to-markdown` on the command line without any Go coding, check out [`html2md`](https://github.com/suntong/html2md#usage), a cli wrapper for `html-to-markdown` that has all the following options and plugins builtin.
## Options
The third parameter to `md.NewConverter` is `*md.Options`.
For example you can change the character that is around a bold text ("`**`") to a different one (for example "`__`") by changing the value of `StrongDelimiter`.
```go
opt := &md.Options{
StrongDelimiter: "__", // default: **
// ...
}
converter := md.NewConverter("", true, opt)
```
For all the possible options look at [godocs](https://godoc.org/github.com/JohannesKaufmann/html-to-markdown/#Options) and for a example look at the [example](/examples/options/main.go).
## Adding Rules
```go
converter.AddRules(
md.Rule{
Filter: []string{"del", "s", "strike"},
Replacement: func(content string, selec *goquery.Selection, opt *md.Options) *string {
// You need to return a pointer to a string (md.String is just a helper function).
// If you return nil the next function for that html element
// will be picked. For example you could only convert an element
// if it has a certain class name and fallback if not.
content = strings.TrimSpace(content)
return md.String("~" + content + "~")
},
},
// more rules
)
```
For more information have a look at the example [add_rules](/examples/add_rules/main.go).
## Using Plugins
If you want plugins (github flavored markdown like striketrough, tables, ...) you can pass it to `Use`.
```go
import "github.com/JohannesKaufmann/html-to-markdown/plugin"
// Use the `GitHubFlavored` plugin from the `plugin` package.
converter.Use(plugin.GitHubFlavored())
```
Or if you only want to use the `Strikethrough` plugin. You can change the character that distinguishes
the text that is crossed out by setting the first argument to a different value (for example "~~" instead of "~").
```go
converter.Use(plugin.Strikethrough(""))
```
For more information have a look at the example [github_flavored](/examples/github_flavored/main.go).
---
These are the plugins located in the [plugin folder](/plugin) which you can use by importing "github.com/JohannesKaufmann/html-to-markdown/plugin".
| Name | Description |
| --------------------- | ------------------------------------------------------------------------------------------- |
| GitHubFlavored | GitHub's Flavored Markdown contains `TaskListItems`, `Strikethrough` and `Table`. |
| TaskListItems | (Included in `GitHubFlavored`). Converts `` checkboxes into `- [x] Task`. |
| Strikethrough | (Included in `GitHubFlavored`). Converts ``, ``, and `` to the `~~` syntax. |
| Table | (Included in `GitHubFlavored`). Convert a `
` into something like this... |
| TableCompat | |
| | |
| VimeoEmbed | |
| YoutubeEmbed | |
| | |
| ConfluenceCodeBlock | Converts `` elements that are used in Atlassian’s Wiki "Confluence". |
| ConfluenceAttachments | Converts `` elements. |
These are the plugins in other repositories:
| Name | Description |
| ---------------------------- | ------------------- |
| \[Plugin Name\]\(Your Link\) | A short description |
I you write a plugin, feel free to open a PR that adds your Plugin to this list.
## Writing Plugins
Have a look at the [plugin folder](/plugin) for a reference implementation. The most basic one is [Strikethrough](/plugin/strikethrough.go).
## Security
This library produces markdown that is readable and can be changed by humans.
Once you convert this markdown back to HTML (e.g. using [goldmark](https://github.com/yuin/goldmark) or [blackfriday](https://github.com/russross/blackfriday)) you need to be careful of malicious content.
This library does NOT sanitize untrusted content. Use an HTML sanitizer such as [bluemonday](https://github.com/microcosm-cc/bluemonday) before displaying the HTML in the browser.
## Other Methods
[Godoc](https://godoc.org/github.com/JohannesKaufmann/html-to-markdown)
### `func (c *Converter) Keep(tags ...string) *Converter`
Determines which elements are to be kept and rendered as HTML.
### `func (c *Converter) Remove(tags ...string) *Converter`
Determines which elements are to be removed altogether i.e. converted to an empty string.
## Escaping
Some characters have a special meaning in markdown. For example, the character "\*" can be used for lists, emphasis and dividers. By placing a backlash before that character (e.g. "\\\*") you can "escape" it. Then the character will render as a raw "\*" without the _"markdown meaning"_ applied.
But why is "escaping" even necessary?
```md
Paragraph 1
-
Paragraph 2
```
The markdown above doesn't seem that problematic. But "Paragraph 1" (with only one hyphen below) will be recognized as a _setext heading_.
```html
Paragraph 1
Paragraph 2
```
A well-placed backslash character would prevent that...
```md
Paragraph 1
\-
Paragraph 2
```
---
How to configure escaping? Depending on the `EscapeMode` option, the markdown output is going to be different.
```go
opt = &md.Options{
EscapeMode: "basic", // default
}
```
Lets try it out with this HTML input:
| | |
| -------- | ----------------------------------------------------- |
| input | `
fake **bold** and real bold
` |
| | |
| | **With EscapeMode "basic"** |
| output | `fake \*\*bold\*\* and real **bold**` |
| rendered | fake \*\*bold\*\* and real **bold** |
| | |
| | **With EscapeMode "disabled"** |
| output | `fake **bold** and real **bold**` |
| rendered | fake **bold** and real **bold** |
With **basic** escaping, we get some escape characters (the backlash "\\") but it renders correctly.
With escaping **disabled**, the fake and real bold can't be distinguished in the markdown. That means it is both going to render as bold.
---
So now you know the purpose of escaping. However, if you encounter some content where the escaping breaks, you can manually disable it. But please also open an issue!
## Issues
If you find HTML snippets (or even full websites) that don't produce the expected results, please open an issue!
## Contributing & Testing
Please first discuss the change you wish to make, by opening an issue. I'm also happy to guide you to where a change is most likely needed.
_Note: The outside API should not change because of backwards compatibility..._
You don't have to be afraid of breaking the converter, since there are many "Golden File Tests":
Add your problematic HTML snippet to one of the `input.html` files in the `testdata` folder. Then run `go test -update` and have a look at which `.golden` files changed in GIT.
You can now change the internal logic and inspect what impact your change has by running `go test -update` again.
_Note: Before submitting your change as a PR, make sure that you run those tests and check the files into GIT..._
## Related Projects
- [turndown (js)](https://github.com/domchristie/turndown), a very good library written in javascript.
- [lunny/html2md](https://github.com/lunny/html2md), which is using [regex instead of goquery](https://stackoverflow.com/a/1732454). I came around a few edge case when using it (leaving some html comments, ...) so I wrote my own.
## License
This project is licensed under the terms of the MIT license.
golang-github-johanneskaufmann-html-to-markdown-1.6.0/SECURITY.md 0000664 0000000 0000000 00000000332 14624642056 0024573 0 ustar 00root root 0000000 0000000 # Security Policy
## Reporting a Vulnerability
Please report (suspected) security vulnerabilities to johannes@joina.de with the subject _"Security html-to-markdown"_ and you will receive a response within 48 hours.
golang-github-johanneskaufmann-html-to-markdown-1.6.0/commonmark.go 0000664 0000000 0000000 00000026630 14624642056 0025505 0 ustar 00root root 0000000 0000000 package md
import (
"fmt"
"unicode"
"regexp"
"strconv"
"strings"
"unicode/utf8"
"github.com/JohannesKaufmann/html-to-markdown/escape"
"github.com/PuerkitoBio/goquery"
)
var multipleSpacesR = regexp.MustCompile(` +`)
var commonmark = []Rule{
{
Filter: []string{"ul", "ol"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
parent := selec.Parent()
// we have a nested list, were the ul/ol is inside a list item
// -> based on work done by @requilence from @anytypeio
if (parent.Is("li") || parent.Is("ul") || parent.Is("ol")) && parent.Children().Last().IsSelection(selec) {
// add a line break prefix if the parent's text node doesn't have it.
// that makes sure that every list item is on its on line
lastContentTextNode := strings.TrimRight(parent.Nodes[0].FirstChild.Data, " \t")
if !strings.HasSuffix(lastContentTextNode, "\n") {
content = "\n" + content
}
// remove empty lines between lists
trimmedSpaceContent := strings.TrimRight(content, " \t")
if strings.HasSuffix(trimmedSpaceContent, "\n") {
content = strings.TrimRightFunc(content, unicode.IsSpace)
}
} else {
content = "\n\n" + content + "\n\n"
}
return &content
},
},
{
Filter: []string{"li"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
if strings.TrimSpace(content) == "" {
return nil
}
// remove leading newlines
content = leadingNewlinesR.ReplaceAllString(content, "")
// replace trailing newlines with just a single one
content = trailingNewlinesR.ReplaceAllString(content, "\n")
// remove leading spaces
content = strings.TrimLeft(content, " ")
prefix := selec.AttrOr(attrListPrefix, "")
// `prefixCount` is not nessesarily the length of the empty string `prefix`
// but how much space is reserved for the prefixes of the siblings.
prefixCount, previousPrefixCounts := countListParents(opt, selec)
// if the prefix is not needed, balance it by adding the usual prefix spaces
if prefix == "" {
prefix = strings.Repeat(" ", prefixCount)
}
// indent the prefix so that the nested links are represented
indent := strings.Repeat(" ", previousPrefixCounts)
prefix = indent + prefix
content = IndentMultiLineListItem(opt, content, prefixCount+previousPrefixCounts)
return String(prefix + content + "\n")
},
},
{
Filter: []string{"#text"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
text := selec.Text()
if trimmed := strings.TrimSpace(text); trimmed == "" {
return String("")
}
text = tabR.ReplaceAllString(text, " ")
// replace multiple spaces by one space: dont accidentally make
// normal text be indented and thus be a code block.
text = multipleSpacesR.ReplaceAllString(text, " ")
if opt.EscapeMode == "basic" {
text = escape.MarkdownCharacters(text)
}
// if its inside a list, trim the spaces to not mess up the indentation
parent := selec.Parent()
next := selec.Next()
if IndexWithText(selec) == 0 &&
(parent.Is("li") || parent.Is("ol") || parent.Is("ul")) &&
(next.Is("ul") || next.Is("ol")) {
// trim only spaces and not new lines
text = strings.Trim(text, ` `)
}
return &text
},
},
{
Filter: []string{"p", "div"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
parent := goquery.NodeName(selec.Parent())
if IsInlineElement(parent) || parent == "li" {
content = "\n" + content + "\n"
return &content
}
// remove unnecessary spaces to have clean markdown
content = TrimpLeadingSpaces(content)
content = "\n\n" + content + "\n\n"
return &content
},
},
{
Filter: []string{"h1", "h2", "h3", "h4", "h5", "h6"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
if strings.TrimSpace(content) == "" {
return nil
}
content = strings.Replace(content, "\n", " ", -1)
content = strings.Replace(content, "\r", " ", -1)
content = strings.Replace(content, `#`, `\#`, -1)
content = strings.TrimSpace(content)
insideLink := selec.ParentsFiltered("a").Length() > 0
if insideLink {
text := opt.StrongDelimiter + content + opt.StrongDelimiter
text = AddSpaceIfNessesary(selec, text)
return &text
}
node := goquery.NodeName(selec)
level, err := strconv.Atoi(node[1:])
if err != nil {
return nil
}
if opt.HeadingStyle == "setext" && level < 3 {
line := "-"
if level == 1 {
line = "="
}
underline := strings.Repeat(line, len(content))
return String("\n\n" + content + "\n" + underline + "\n\n")
}
prefix := strings.Repeat("#", level)
text := "\n\n" + prefix + " " + content + "\n\n"
return &text
},
},
{
Filter: []string{"strong", "b"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
// only use one bold tag if they are nested
parent := selec.Parent()
if parent.Is("strong") || parent.Is("b") {
return &content
}
trimmed := strings.TrimSpace(content)
if trimmed == "" {
return &trimmed
}
// If there is a newline character between the start and end delimiter
// the delimiters won't be recognized. Either we remove all newline characters
// OR on _every_ line we put start & end delimiters.
trimmed = delimiterForEveryLine(trimmed, opt.StrongDelimiter)
// Always have a space to the side to recognize the delimiter
trimmed = AddSpaceIfNessesary(selec, trimmed)
return &trimmed
},
},
{
Filter: []string{"i", "em"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
// only use one italic tag if they are nested
parent := selec.Parent()
if parent.Is("i") || parent.Is("em") {
return &content
}
trimmed := strings.TrimSpace(content)
if trimmed == "" {
return &trimmed
}
// If there is a newline character between the start and end delimiter
// the delimiters won't be recognized. Either we remove all newline characters
// OR on _every_ line we put start & end delimiters.
trimmed = delimiterForEveryLine(trimmed, opt.EmDelimiter)
// Always have a space to the side to recognize the delimiter
trimmed = AddSpaceIfNessesary(selec, trimmed)
return &trimmed
},
},
{
Filter: []string{"img"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
src := selec.AttrOr("src", "")
src = strings.TrimSpace(src)
if src == "" {
return String("")
}
src = opt.GetAbsoluteURL(selec, src, opt.domain)
alt := selec.AttrOr("alt", "")
alt = strings.Replace(alt, "\n", " ", -1)
text := fmt.Sprintf("", alt, src)
return &text
},
},
{
Filter: []string{"a"},
AdvancedReplacement: func(content string, selec *goquery.Selection, opt *Options) (AdvancedResult, bool) {
// if there is no href, no link is used. So just return the content inside the link
href, ok := selec.Attr("href")
if !ok || strings.TrimSpace(href) == "" || strings.TrimSpace(href) == "#" {
return AdvancedResult{
Markdown: content,
}, false
}
href = opt.GetAbsoluteURL(selec, href, opt.domain)
// having multiline content inside a link is a bit tricky
content = EscapeMultiLine(content)
var title string
if t, ok := selec.Attr("title"); ok {
t = strings.Replace(t, "\n", " ", -1)
// escape all quotes
t = strings.Replace(t, `"`, `\"`, -1)
title = fmt.Sprintf(` "%s"`, t)
}
// if there is no link content (for example because it contains an svg)
// the 'title' or 'aria-label' attribute is used instead.
if strings.TrimSpace(content) == "" {
content = selec.AttrOr("title", selec.AttrOr("aria-label", ""))
}
// a link without text won't de displayed anyway
if content == "" {
return AdvancedResult{}, true
}
if opt.LinkStyle == "inlined" {
md := fmt.Sprintf("[%s](%s%s)", content, href, title)
md = AddSpaceIfNessesary(selec, md)
return AdvancedResult{
Markdown: md,
}, false
}
var replacement string
var reference string
switch opt.LinkReferenceStyle {
case "collapsed":
replacement = "[" + content + "][]"
reference = "[" + content + "]: " + href + title
case "shortcut":
replacement = "[" + content + "]"
reference = "[" + content + "]: " + href + title
default:
id := selec.AttrOr("data-index", "")
replacement = "[" + content + "][" + id + "]"
reference = "[" + id + "]: " + href + title
}
replacement = AddSpaceIfNessesary(selec, replacement)
return AdvancedResult{Markdown: replacement, Footer: reference}, false
},
},
{
Filter: []string{"code", "kbd", "samp", "tt"},
Replacement: func(_ string, selec *goquery.Selection, opt *Options) *string {
code := getCodeContent(selec)
// Newlines in the text aren't great, since this is inline code and not a code block.
// Newlines will be stripped anyway in the browser, but it won't be recognized as code
// from the markdown parser when there is more than one newline.
// So limit to
code = multipleNewLinesRegex.ReplaceAllString(code, "\n")
fenceChar := '`'
maxCount := calculateCodeFenceOccurrences(fenceChar, code)
maxCount++
fence := strings.Repeat(string(fenceChar), maxCount)
// code block contains a backtick as first character
if strings.HasPrefix(code, "`") {
code = " " + code
}
// code block contains a backtick as last character
if strings.HasSuffix(code, "`") {
code = code + " "
}
// TODO: configure delimeter in options?
text := fence + code + fence
text = AddSpaceIfNessesary(selec, text)
return &text
},
},
{
Filter: []string{"pre"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
codeElement := selec.Find("code")
language := codeElement.AttrOr("class", "")
language = strings.Replace(language, "language-", "", 1)
code := getCodeContent(selec)
fenceChar, _ := utf8.DecodeRuneInString(opt.Fence)
fence := CalculateCodeFence(fenceChar, code)
text := "\n\n" + fence + language + "\n" +
code +
"\n" + fence + "\n\n"
return &text
},
},
{
Filter: []string{"hr"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
// e.g. `## --- Heading` would look weird, so don't render a divider if inside a heading
insideHeading := selec.ParentsFiltered("h1,h2,h3,h4,h5,h6").Length() > 0
if insideHeading {
return String("")
}
text := "\n\n" + opt.HorizontalRule + "\n\n"
return &text
},
},
{
Filter: []string{"br"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
return String("\n\n")
},
},
{
Filter: []string{"blockquote"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
content = strings.TrimSpace(content)
if content == "" {
return nil
}
content = multipleNewLinesRegex.ReplaceAllString(content, "\n\n")
var beginningR = regexp.MustCompile(`(?m)^`)
content = beginningR.ReplaceAllString(content, "> ")
text := "\n\n" + content + "\n\n"
return &text
},
},
{
Filter: []string{"noscript"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
// for now remove the contents of noscript. But in the future we could
// tell goquery to parse the contents of the tag.
// -> https://github.com/PuerkitoBio/goquery/issues/139#issuecomment-517526070
return nil
},
},
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/commonmark_test.go 0000664 0000000 0000000 00000014611 14624642056 0026540 0 ustar 00root root 0000000 0000000 package md_test
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path"
"path/filepath"
"strings"
"testing"
md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
"github.com/sebdah/goldie/v2"
"github.com/yuin/goldmark"
"github.com/yuin/goldmark/extension"
)
type Variation struct {
Options *md.Options
Plugins []md.Plugin
}
type GoldenTest struct {
Name string
Domain string
DisableGoldmark bool
Variations map[string]Variation
}
func runGoldenTest(t *testing.T, test GoldenTest, variationKey string) {
variation := test.Variations[variationKey]
g := goldie.New(t)
// testdata/TestCommonmark/name/input.html
p := path.Join(t.Name(), "input.html")
// get the input html from a file
input, err := ioutil.ReadFile(path.Join("testdata", p))
if err != nil {
t.Error(err)
return
}
if test.Domain == "" {
test.Domain = "example.com"
}
conv := md.NewConverter(test.Domain, true, variation.Options)
conv.Keep("keep-tag").Remove("remove-tag")
for _, plugin := range variation.Plugins {
conv.Use(plugin)
}
markdown, err := conv.ConvertBytes(input)
if err != nil {
t.Error(err)
}
// testdata/TestCommonmark/name/output.default.golden
p = path.Join(t.Name(), "output."+variationKey)
g.Assert(t, p, markdown)
gold := goldmark.New(goldmark.WithExtensions(extension.GFM))
var buf bytes.Buffer
if err := gold.Convert(markdown, &buf); err != nil {
t.Error(err)
}
if !test.DisableGoldmark {
// testdata/TestCommonmark/name/goldmark.golden
p = path.Join(t.Name(), "goldmark")
g.Assert(t, p, buf.Bytes())
}
}
func RunGoldenTest(t *testing.T, tests []GoldenTest) {
// loop through all test cases that were added manually
dirs := make(map[string]struct{})
for _, test := range tests {
name := test.Name
name = strings.Replace(name, " ", "_", -1)
dirs[name] = struct{}{}
}
// now add all tests that were found on disk to the tests slice
err := filepath.Walk(path.Join("testdata", t.Name()),
func(p string, info os.FileInfo, err error) error {
if err != nil {
return err
}
if !info.IsDir() {
return nil
}
// skip folders that don't contain an input.html file
if _, err := os.Stat(path.Join(p, "input.html")); os.IsNotExist(err) {
return nil
}
parts := strings.SplitN(p, string(os.PathSeparator), 3)
p = parts[2] // remove "testdata/TestCommonmark/" from "testdata/TestCommonmark/..."
_, ok := dirs[p]
if ok {
return nil
}
// add the folder from disk to the tests slice, since its not it there yet
tests = append(tests, GoldenTest{
Name: p,
})
return nil
})
if err != nil {
t.Error(err)
return
}
for _, test := range tests {
if len(test.Variations) == 0 {
test.Variations = map[string]Variation{
"default": {},
}
}
t.Run(test.Name, func(t *testing.T) {
if strings.Contains(t.Name(), "#") {
fmt.Println("the name", test.Name, t.Name(), "seems too be used for multiple tests")
return
}
for variationKey := range test.Variations {
runGoldenTest(t, test, variationKey)
}
})
}
}
func TestCommonmark(t *testing.T) {
var tests = []GoldenTest{
{
Name: "link",
DisableGoldmark: true,
Variations: map[string]Variation{
"relative": {
Options: &md.Options{
GetAbsoluteURL: func(selec *goquery.Selection, rawURL string, domain string) string {
return rawURL
},
},
},
"inlined": {
Options: &md.Options{LinkStyle: "inlined"},
},
"referenced_full": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "full"},
},
"referenced_collapsed": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "collapsed"},
},
"referenced_shortcut": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "shortcut"},
},
},
},
{
Name: "heading",
Variations: map[string]Variation{
"atx": {
Options: &md.Options{HeadingStyle: "atx"},
},
"setext": {
Options: &md.Options{HeadingStyle: "setext"},
},
},
},
{
Name: "italic",
Variations: map[string]Variation{
"asterisks": {
Options: &md.Options{EmDelimiter: "*"},
},
"underscores": {
Options: &md.Options{EmDelimiter: "_"},
},
},
},
{
Name: "bold",
Variations: map[string]Variation{
"asterisks": {
Options: &md.Options{StrongDelimiter: "**"},
},
"underscores": {
Options: &md.Options{StrongDelimiter: "__"},
},
},
},
{
Name: "pre_code",
Variations: map[string]Variation{
"indented": {
Options: &md.Options{CodeBlockStyle: "indented"},
},
"fenced_backtick": {
Options: &md.Options{CodeBlockStyle: "fenced", Fence: "```"},
},
"fenced_tilde": {
Options: &md.Options{CodeBlockStyle: "fenced", Fence: "~~~"},
},
},
},
{
Name: "list",
Variations: map[string]Variation{
"asterisks": {
Options: &md.Options{BulletListMarker: "*"},
},
"dash": {
Options: &md.Options{BulletListMarker: "-"},
},
"plus": {
Options: &md.Options{BulletListMarker: "+"},
},
},
},
{
Name: "list_nested",
DisableGoldmark: true,
Variations: map[string]Variation{
"asterisks": {
Options: &md.Options{BulletListMarker: "*"},
},
"dash": {
Options: &md.Options{BulletListMarker: "-"},
},
"plus": {
Options: &md.Options{BulletListMarker: "+"},
},
},
},
// + all the test on disk that are added automatically
}
RunGoldenTest(t, tests)
}
func TestRealWorld(t *testing.T) {
var tests = []GoldenTest{
{
Name: "blog.golang.org",
Domain: "blog.golang.org",
Variations: map[string]Variation{
"inlined": {
Options: &md.Options{LinkStyle: "inlined"},
},
"referenced_full": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "full"},
},
"referenced_collapsed": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "collapsed"},
},
"referenced_shortcut": {
Options: &md.Options{LinkStyle: "referenced", LinkReferenceStyle: "shortcut"},
},
"emphasis_asterisks": {
Options: &md.Options{EmDelimiter: "*", StrongDelimiter: "**"},
},
"emphasis_underscores": {
Options: &md.Options{EmDelimiter: "_", StrongDelimiter: "__"},
},
},
},
{
Name: "golang.org",
Domain: "golang.org",
},
// + all the test on disk that are added automatically
}
RunGoldenTest(t, tests)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/escape/ 0000775 0000000 0000000 00000000000 14624642056 0024244 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/escape/escape.go 0000664 0000000 0000000 00000003522 14624642056 0026035 0 ustar 00root root 0000000 0000000 // Package escape escapes characters that are commonly used in
// markdown like the * for strong/italic.
package escape
import (
"regexp"
"strings"
)
var backslash = regexp.MustCompile(`\\(\S)`)
var heading = regexp.MustCompile(`(?m)^(#{1,6} )`)
var orderedList = regexp.MustCompile(`(?m)^(\W* {0,3})(\d+)\. `)
var unorderedList = regexp.MustCompile(`(?m)^([^\\\w]*)[*+-] `)
var horizontalDivider = regexp.MustCompile(`(?m)^([-*_] *){3,}$`)
var blockquote = regexp.MustCompile(`(?m)^(\W* {0,3})> `)
var link = regexp.MustCompile(`([\[\]])`)
var replacer = strings.NewReplacer(
`*`, `\*`,
`_`, `\_`,
"`", "\\`",
`|`, `\|`,
)
// MarkdownCharacters escapes common markdown characters so that
// `
**Not Bold**
ends up as correct markdown `\*\*Not Strong\*\*`.
// No worry, the escaped characters will display fine, just without the formatting.
func MarkdownCharacters(text string) string {
// Escape backslash escapes!
text = backslash.ReplaceAllString(text, `\\$1`)
// Escape headings
text = heading.ReplaceAllString(text, `\$1`)
// Escape hr
text = horizontalDivider.ReplaceAllStringFunc(text, func(t string) string {
if strings.Contains(t, "-") {
return strings.Replace(t, "-", `\-`, 3)
} else if strings.Contains(t, "_") {
return strings.Replace(t, "_", `\_`, 3)
}
return strings.Replace(t, "*", `\*`, 3)
})
// Escape ol bullet points
text = orderedList.ReplaceAllString(text, `$1$2\. `)
// Escape ul bullet points
text = unorderedList.ReplaceAllStringFunc(text, func(t string) string {
return regexp.MustCompile(`([*+-])`).ReplaceAllString(t, `\$1`)
})
// Escape blockquote indents
text = blockquote.ReplaceAllString(text, `$1\> `)
// Escape em/strong *
// Escape em/strong _
// Escape code _
text = replacer.Replace(text)
// Escape link & image brackets
text = link.ReplaceAllString(text, `\$1`)
return text
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/ 0000775 0000000 0000000 00000000000 14624642056 0024622 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/add_rules/ 0000775 0000000 0000000 00000000000 14624642056 0026564 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/add_rules/main.go 0000664 0000000 0000000 00000002554 14624642056 0030045 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"log"
"strings"
md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
)
func main() {
html := `Good soundtrack and cake.`
// -> `Good soundtrack ~~and cake~~.`
/*
We want to add a rule when a `span` tag has a class of `bb_strike`.
Have a look at `plugin/strikethrough.go` to see how it is implemented normally.
*/
strikethrough := md.Rule{
Filter: []string{"span"},
Replacement: func(content string, selec *goquery.Selection, opt *md.Options) *string {
// If the span element has not the classname `bb_strike` return nil.
// That way the next rules will apply. In this case the commonmark rules.
// -> return nil -> next rule applies
if !selec.HasClass("bb_strike") {
return nil
}
// Trim spaces so that the following does NOT happen: `~ and cake~`.
// Because of the space it is not recognized as strikethrough.
// -> trim spaces at begin&end of string when inside strong/italic/...
content = strings.TrimSpace(content)
return md.String("~~" + content + "~~")
},
}
conv := md.NewConverter("", true, nil)
conv.AddRules(strikethrough)
// -> add 1+ rules to the converter. the last added will be used first.
markdown, err := conv.ConvertString(html)
if err != nil {
log.Fatal(err)
}
fmt.Printf("\n\nmarkdown:'%s'\n", markdown)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/custom_tag/ 0000775 0000000 0000000 00000000000 14624642056 0026767 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/custom_tag/main.go 0000664 0000000 0000000 00000002200 14624642056 0030234 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"log"
"strings"
md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
)
func main() {
html := `https://youtu.be/1SoMeViDhttps://youtu.be/2SoMeViDhttps://youtu.be/3SoMeViDhttps://youtu.be/4SoMeViDhttps://youtu.be/5SoMeViD
`
videoRule := md.Rule{
// We want to add a rule for a `my_video` tag.
Filter: []string{"my_video"},
Replacement: func(content string, selec *goquery.Selection, opt *md.Options) *string {
text := "click to watch video"
// in this case, the content inside the tag is the url
href := strings.TrimSpace(content)
// format it, so that its `[click to watch video](https://youtu.be/1SoMeViD)\n\n`
md := fmt.Sprintf("[%s](%s)\n\n", text, href)
return &md
},
}
conv := md.NewConverter("", true, nil)
conv.AddRules(videoRule)
// -> add 1+ rules to the converter. the last added will be used first.
markdown, err := conv.ConvertString(html)
if err != nil {
log.Fatal(err)
}
fmt.Printf("\n\nresult:'%s'\n", markdown)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/escaping/ 0000775 0000000 0000000 00000000000 14624642056 0026413 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/escaping/main.go 0000664 0000000 0000000 00000001646 14624642056 0027675 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"log"
md "github.com/JohannesKaufmann/html-to-markdown"
)
func main() {
html := `
fake **bold** and real bold
`
// With "basic" we get:
// "fake \*\*bold\*\* and real **bold**"
// which would render as:
// "
fake **bold** and real bold
"
// With "none" we get:
// "fake **bold** and real **bold**"
// which would render as:
// "
`
/*
- [x] Checked!
- [ ] Check Me!
*/
conv := md.NewConverter("", true, nil)
// Use the `GitHubFlavored` plugin from the `plugin` package.
conv.Use(plugin.GitHubFlavored())
markdown, err := conv.ConvertString(html)
if err != nil {
log.Fatal(err)
}
fmt.Println(markdown)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/goquery/ 0000775 0000000 0000000 00000000000 14624642056 0026315 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/goquery/main.go 0000664 0000000 0000000 00000000656 14624642056 0027577 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"log"
md "github.com/JohannesKaufmann/html-to-markdown"
"github.com/PuerkitoBio/goquery"
)
func main() {
url := "https://blog.golang.org/godoc-documenting-go-code"
doc, err := goquery.NewDocument(url)
if err != nil {
log.Fatal(err)
}
content := doc.Find("#content")
conv := md.NewConverter(md.DomainFromURL(url), true, nil)
markdown := conv.Convert(content)
fmt.Println(markdown)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/options/ 0000775 0000000 0000000 00000000000 14624642056 0026315 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/examples/options/main.go 0000664 0000000 0000000 00000000631 14624642056 0027570 0 ustar 00root root 0000000 0000000 package main
import (
"fmt"
"log"
md "github.com/JohannesKaufmann/html-to-markdown"
)
func main() {
html := `Bold Text`
// -> `__Bold Text__`
// instead of `**Bold Text**`
opt := &md.Options{
StrongDelimiter: "__", // default: **
}
conv := md.NewConverter("", true, opt)
markdown, err := conv.ConvertString(html)
if err != nil {
log.Fatal(err)
}
fmt.Println(markdown)
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/from.go 0000664 0000000 0000000 00000027440 14624642056 0024305 0 ustar 00root root 0000000 0000000 // Package md converts html to markdown.
//
// converter := md.NewConverter("", true, nil)
//
// html = `Important`
//
// markdown, err := converter.ConvertString(html)
// if err != nil {
// log.Fatal(err)
// }
// fmt.Println("md ->", markdown)
// Or if you are already using goquery:
// markdown, err := converter.Convert(selec)
package md
import (
"bytes"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"regexp"
"strconv"
"strings"
"sync"
"time"
"github.com/PuerkitoBio/goquery"
)
type simpleRuleFunc func(content string, selec *goquery.Selection, options *Options) *string
type ruleFunc func(content string, selec *goquery.Selection, options *Options) (res AdvancedResult, skip bool)
// BeforeHook runs before the converter and can be used to transform the original html
type BeforeHook func(selec *goquery.Selection)
// Afterhook runs after the converter and can be used to transform the resulting markdown
type Afterhook func(markdown string) string
// Converter is initialized by NewConverter.
type Converter struct {
mutex sync.RWMutex
rules map[string][]ruleFunc
keep map[string]struct{}
remove map[string]struct{}
before []BeforeHook
after []Afterhook
domain string
options Options
}
func validate(val string, possible ...string) error {
for _, e := range possible {
if e == val {
return nil
}
}
return fmt.Errorf("field must be one of %v but got %s", possible, val)
}
func validateOptions(opt Options) error {
if err := validate(opt.HeadingStyle, "setext", "atx"); err != nil {
return err
}
if strings.Count(opt.HorizontalRule, "*") < 3 &&
strings.Count(opt.HorizontalRule, "_") < 3 &&
strings.Count(opt.HorizontalRule, "-") < 3 {
return errors.New("HorizontalRule must be at least 3 characters of '*', '_' or '-' but got " + opt.HorizontalRule)
}
if err := validate(opt.BulletListMarker, "-", "+", "*"); err != nil {
return err
}
if err := validate(opt.CodeBlockStyle, "indented", "fenced"); err != nil {
return err
}
if err := validate(opt.Fence, "```", "~~~"); err != nil {
return err
}
if err := validate(opt.EmDelimiter, "_", "*"); err != nil {
return err
}
if err := validate(opt.StrongDelimiter, "**", "__"); err != nil {
return err
}
if err := validate(opt.LinkStyle, "inlined", "referenced"); err != nil {
return err
}
if err := validate(opt.LinkReferenceStyle, "full", "collapsed", "shortcut"); err != nil {
return err
}
return nil
}
var (
attrListPrefix = "data-converter-list-prefix"
)
// NewConverter initializes a new converter and holds all the rules.
// - `domain` is used for links and images to convert relative urls ("/image.png") to absolute urls.
// - CommonMark is the default set of rules. Set enableCommonmark to false if you want
// to customize everything using AddRules and DONT want to fallback to default rules.
func NewConverter(domain string, enableCommonmark bool, options *Options) *Converter {
conv := &Converter{
domain: domain,
rules: make(map[string][]ruleFunc),
keep: make(map[string]struct{}),
remove: make(map[string]struct{}),
}
conv.before = append(conv.before, func(selec *goquery.Selection) {
selec.Find("a[href]").Each(func(i int, s *goquery.Selection) {
// TODO: don't hardcode "data-index" and rename it to avoid accidental conflicts
s.SetAttr("data-index", strconv.Itoa(i+1))
})
})
conv.before = append(conv.before, func(selec *goquery.Selection) {
selec.Find("li").Each(func(i int, s *goquery.Selection) {
prefix := getListPrefix(options, s)
s.SetAttr(attrListPrefix, prefix)
})
})
conv.after = append(conv.after, func(markdown string) string {
markdown = strings.TrimSpace(markdown)
markdown = multipleNewLinesRegex.ReplaceAllString(markdown, "\n\n")
// remove unnecessary trailing spaces to have clean markdown
markdown = TrimTrailingSpaces(markdown)
return markdown
})
if enableCommonmark {
conv.AddRules(commonmark...)
conv.remove["script"] = struct{}{}
conv.remove["style"] = struct{}{}
conv.remove["textarea"] = struct{}{}
}
// TODO: put domain in options?
if options == nil {
options = &Options{}
}
if options.HeadingStyle == "" {
options.HeadingStyle = "atx"
}
if options.HorizontalRule == "" {
options.HorizontalRule = "* * *"
}
if options.BulletListMarker == "" {
options.BulletListMarker = "-"
}
if options.CodeBlockStyle == "" {
options.CodeBlockStyle = "indented"
}
if options.Fence == "" {
options.Fence = "```"
}
if options.EmDelimiter == "" {
options.EmDelimiter = "_"
}
if options.StrongDelimiter == "" {
options.StrongDelimiter = "**"
}
if options.LinkStyle == "" {
options.LinkStyle = "inlined"
}
if options.LinkReferenceStyle == "" {
options.LinkReferenceStyle = "full"
}
if options.EscapeMode == "" {
options.EscapeMode = "basic"
}
// for now, store it in the options
options.domain = domain
if options.GetAbsoluteURL == nil {
options.GetAbsoluteURL = DefaultGetAbsoluteURL
}
conv.options = *options
err := validateOptions(conv.options)
if err != nil {
log.Println("markdown options is not valid:", err)
}
return conv
}
func (conv *Converter) getRuleFuncs(tag string) []ruleFunc {
conv.mutex.RLock()
defer conv.mutex.RUnlock()
r, ok := conv.rules[tag]
if !ok || len(r) == 0 {
if _, keep := conv.keep[tag]; keep {
return []ruleFunc{wrap(ruleKeep)}
}
if _, remove := conv.remove[tag]; remove {
return nil // TODO:
}
return []ruleFunc{wrap(ruleDefault)}
}
return r
}
func wrap(simple simpleRuleFunc) ruleFunc {
return func(content string, selec *goquery.Selection, opt *Options) (AdvancedResult, bool) {
res := simple(content, selec, opt)
if res == nil {
return AdvancedResult{}, true
}
return AdvancedResult{Markdown: *res}, false
}
}
// Before registers a hook that is run before the conversion. It
// can be used to transform the original goquery html document.
//
// For example, the default before hook adds an index to every link,
// so that the `a` tag rule (for "reference" "full") can have an incremental number.
func (conv *Converter) Before(hooks ...BeforeHook) *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
for _, hook := range hooks {
conv.before = append(conv.before, hook)
}
return conv
}
// After registers a hook that is run after the conversion. It
// can be used to transform the markdown document that is about to be returned.
//
// For example, the default after hook trims the returned markdown.
func (conv *Converter) After(hooks ...Afterhook) *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
for _, hook := range hooks {
conv.after = append(conv.after, hook)
}
return conv
}
// ClearBefore clears the current before hooks (including the default before hooks).
func (conv *Converter) ClearBefore() *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
conv.before = nil
return conv
}
// ClearAfter clears the current after hooks (including the default after hooks).
func (conv *Converter) ClearAfter() *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
conv.after = nil
return conv
}
// AddRules adds the rules that are passed in to the converter.
//
// By default it overrides the rule for that html tag. You can
// fall back to the default rule by returning nil.
func (conv *Converter) AddRules(rules ...Rule) *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
for _, rule := range rules {
if len(rule.Filter) == 0 {
log.Println("you need to specify at least one filter for your rule")
}
for _, filter := range rule.Filter {
r, _ := conv.rules[filter]
if rule.AdvancedReplacement != nil {
r = append(r, rule.AdvancedReplacement)
} else {
r = append(r, wrap(rule.Replacement))
}
conv.rules[filter] = r
}
}
return conv
}
// Keep certain html tags in the generated output.
func (conv *Converter) Keep(tags ...string) *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
for _, tag := range tags {
conv.keep[tag] = struct{}{}
}
return conv
}
// Remove certain html tags from the source.
func (conv *Converter) Remove(tags ...string) *Converter {
conv.mutex.Lock()
defer conv.mutex.Unlock()
for _, tag := range tags {
conv.remove[tag] = struct{}{}
}
return conv
}
// Plugin can be used to extends functionality beyond what
// is offered by commonmark.
type Plugin func(conv *Converter) []Rule
// Use can be used to add additional functionality to the converter. It is
// used when its not sufficient to use only rules for example in Plugins.
func (conv *Converter) Use(plugins ...Plugin) *Converter {
for _, plugin := range plugins {
rules := plugin(conv)
conv.AddRules(rules...) // TODO: for better performance only use one lock for all plugins
}
return conv
}
// Timeout for the http client
var Timeout = time.Second * 10
var netClient = &http.Client{
Timeout: Timeout,
}
// DomainFromURL returns `u.Host` from the parsed url.
func DomainFromURL(rawURL string) string {
rawURL = strings.TrimSpace(rawURL)
u, _ := url.Parse(rawURL)
if u != nil && u.Host != "" {
return u.Host
}
// lets try it again by adding a scheme
u, _ = url.Parse("http://" + rawURL)
if u != nil {
return u.Host
}
return ""
}
// Reduce many newline characters `\n` to at most 2 new line characters.
var multipleNewLinesRegex = regexp.MustCompile(`[\n]{2,}`)
// Convert returns the content from a goquery selection.
// If you have a goquery document just pass in doc.Selection.
func (conv *Converter) Convert(selec *goquery.Selection) string {
conv.mutex.RLock()
domain := conv.domain
options := conv.options
l := len(conv.rules)
if l == 0 {
log.Println("you have added no rules. either enable commonmark or add you own.")
}
before := conv.before
after := conv.after
conv.mutex.RUnlock()
// before hook
for _, hook := range before {
hook(selec)
}
res := conv.selecToMD(domain, selec, &options)
markdown := res.Markdown
if res.Header != "" {
markdown = res.Header + "\n\n" + markdown
}
if res.Footer != "" {
markdown += "\n\n" + res.Footer
}
// after hook
for _, hook := range after {
markdown = hook(markdown)
}
return markdown
}
// ConvertReader returns the content from a reader and returns a buffer.
func (conv *Converter) ConvertReader(reader io.Reader) (bytes.Buffer, error) {
var buffer bytes.Buffer
doc, err := goquery.NewDocumentFromReader(reader)
if err != nil {
return buffer, err
}
buffer.WriteString(
conv.Convert(doc.Selection),
)
return buffer, nil
}
// ConvertResponse returns the content from a html response.
func (conv *Converter) ConvertResponse(res *http.Response) (string, error) {
doc, err := goquery.NewDocumentFromResponse(res)
if err != nil {
return "", err
}
return conv.Convert(doc.Selection), nil
}
// ConvertString returns the content from a html string. If you
// already have a goquery selection use `Convert`.
func (conv *Converter) ConvertString(html string) (string, error) {
doc, err := goquery.NewDocumentFromReader(strings.NewReader(html))
if err != nil {
return "", err
}
return conv.Convert(doc.Selection), nil
}
// ConvertBytes returns the content from a html byte array.
func (conv *Converter) ConvertBytes(bytes []byte) ([]byte, error) {
res, err := conv.ConvertString(string(bytes))
if err != nil {
return nil, err
}
return []byte(res), nil
}
// ConvertURL returns the content from the page with that url.
func (conv *Converter) ConvertURL(url string) (string, error) {
// not using goquery.NewDocument directly because of the timeout
resp, err := netClient.Get(url)
if err != nil {
return "", err
}
if resp.StatusCode < 200 || resp.StatusCode > 299 {
return "", fmt.Errorf("expected a status code in the 2xx range but got %d", resp.StatusCode)
}
doc, err := goquery.NewDocumentFromResponse(resp)
if err != nil {
return "", err
}
domain := DomainFromURL(url)
if conv.domain != domain {
log.Printf("expected '%s' as the domain but got '%s' \n", conv.domain, domain)
}
return conv.Convert(doc.Selection), nil
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/from_test.go 0000664 0000000 0000000 00000036173 14624642056 0025347 0 ustar 00root root 0000000 0000000 package md
import (
"bytes"
"errors"
"io/ioutil"
"log"
"net/http"
"net/http/httptest"
"os"
"strings"
"sync"
"testing"
"github.com/PuerkitoBio/goquery"
)
func TestConvertReader(t *testing.T) {
input := `Bold`
expected := `**Bold**`
converter := NewConverter("", true, nil)
res, err := converter.ConvertReader(strings.NewReader(input))
if err != nil {
t.Error(err)
}
if !bytes.Equal(res.Bytes(), []byte(expected)) {
t.Error("the result is different that expected")
}
}
type ErrReader struct{ Error error }
// -> https://stackoverflow.com/a/57452918
func (e *ErrReader) Read([]byte) (int, error) {
return 0, e.Error
}
func TestConvertReader_Error(t *testing.T) {
reader := &ErrReader{
Error: errors.New("we got an error"),
}
converter := NewConverter("", true, nil)
res, err := converter.ConvertReader(reader)
if err != reader.Error {
t.Error("expected an error")
}
if res.Len() != 0 {
t.Error("expected an empty buffer")
}
}
func TestConvertBytes(t *testing.T) {
input := `Bold`
expected := `**Bold**`
converter := NewConverter("", true, nil)
res, err := converter.ConvertBytes([]byte(input))
if err != nil {
t.Error(err)
}
if !bytes.Equal(res, []byte(expected)) {
t.Error("the result is different that expected")
}
}
func TestConvertBytes_Empty(t *testing.T) {
converter := NewConverter("", true, nil)
res, err := converter.ConvertBytes(nil)
if err != nil {
t.Error(err)
}
if !bytes.Equal(res, []byte("")) {
t.Error("the result is different that expected")
}
}
func TestConvertResponse(t *testing.T) {
input := `Bold`
expected := `**Bold**`
converter := NewConverter("", true, nil)
res, err := converter.ConvertResponse(&http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(bytes.NewBufferString(input)),
Request: &http.Request{},
})
if err != nil {
t.Error(err)
}
if res != expected {
t.Error("the result is different that expected")
}
}
func TestConvertResponse_Error(t *testing.T) {
expectedErr := errors.New("custom error reader")
converter := NewConverter("", true, nil)
res, err := converter.ConvertResponse(&http.Response{
StatusCode: 200,
Body: ioutil.NopCloser(&ErrReader{
Error: expectedErr,
}),
Request: &http.Request{},
})
if err != expectedErr {
t.Error(err)
}
if res != "" {
t.Error("the result is different that expected")
}
}
func TestConvertString(t *testing.T) {
input := `Bold`
expected := `**Bold**`
converter := NewConverter("", true, nil)
res, err := converter.ConvertString(input)
if err != nil {
t.Error(err)
}
if res != expected {
t.Error("the result is different that expected")
}
}
func TestConvertSelection(t *testing.T) {
input := `Bold`
expected := `**Bold**`
doc, err := goquery.NewDocumentFromReader(strings.NewReader(input))
if err != nil {
t.Error(err)
}
converter := NewConverter("", true, nil)
res := converter.Convert(doc.Selection)
if res != expected {
t.Error("the result is different that expected")
}
}
func TestConvertURL(t *testing.T) {
input := `Bold`
expected := `**Bold**`
// Start a local HTTP server
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.Write([]byte(input))
}))
// Close the server when test finishes
defer server.Close()
// override the client used in `ConvertURL`
netClient = server.Client()
converter := NewConverter(server.URL, true, nil)
res, err := converter.ConvertURL(server.URL)
if err != nil {
t.Error(err)
}
if res != expected {
t.Error("the result is different that expected")
}
}
func TestConvertURL_Error(t *testing.T) {
url := "abc https://example.com"
converter := NewConverter("", true, nil)
res, err := converter.ConvertURL(url)
if err == nil {
t.Error("expected an error")
}
if res != "" {
t.Error("the result is different that expected")
}
}
func TestConvertURL_ErrorStatusCode(t *testing.T) {
// Start a local HTTP server
server := httptest.NewServer(http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
rw.WriteHeader(http.StatusNotFound)
rw.Write([]byte("404 Not Found"))
}))
// Close the server when test finishes
defer server.Close()
// override the client used in `ConvertURL`
netClient = server.Client()
converter := NewConverter(server.URL, true, nil)
res, err := converter.ConvertURL(server.URL)
if err == nil {
t.Error("expected an error")
}
if res != "" {
t.Error("the result is different that expected")
}
}
// - - - - - - - - - - - - //
func TestNewConverter_NoRules(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
log.SetFlags(0)
defer func() {
// reset the options back to the defaults
log.SetOutput(os.Stderr)
log.SetFlags(3)
}()
input := `Bold`
expected := ``
// disable commonmark
converter := NewConverter("", false, nil)
res, err := converter.ConvertString(input)
if err != nil {
t.Error(err)
}
if res != expected {
t.Error("the result is different that expected")
}
if strings.TrimSuffix(buf.String(), "\n") != "you have added no rules. either enable commonmark or add you own." {
t.Error("expected a different log message")
}
}
func TestNewConverter_ValidateOptions(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
log.SetFlags(0)
defer func() {
// reset the options back to the defaults
log.SetOutput(os.Stderr)
log.SetFlags(3)
}()
input := `Bold`
expected := `====Bold====`
converter := NewConverter("", true, &Options{
StrongDelimiter: "====",
})
res, err := converter.ConvertString(input)
if err != nil {
t.Error(err)
}
if res != expected {
t.Error("the result is different that expected")
}
if strings.TrimSuffix(buf.String(), "\n") != "markdown options is not valid: field must be one of [** __] but got ====" {
t.Error("expected a different log message")
}
}
func TestNewConverter_ValidateOptions_All(t *testing.T) {
var tests = []struct {
name string
options *Options
input string
expected string
}{
{
name: "HeadingStyle",
options: &Options{
HeadingStyle: "invalid",
},
input: `
`,
expected: "^^^\ntest\n^^^",
},
{
name: "EmDelimiter",
options: &Options{
EmDelimiter: "-",
},
input: `test`,
expected: "-test-",
},
{
name: "LinkStyle",
options: &Options{
LinkStyle: "invalid",
},
input: `link`,
expected: `[link][1]
[1]: example.com`,
},
{
name: "LinkReferenceStyle",
options: &Options{
LinkReferenceStyle: "invalid",
},
input: `link`,
expected: "[link](example.com)",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
log.SetFlags(0)
defer func() {
// reset the options back to the defaults
log.SetOutput(os.Stderr)
log.SetFlags(3)
}()
converter := NewConverter("", true, test.options)
res, err := converter.ConvertString(test.input)
if err != nil {
t.Error(err)
}
if res != test.expected {
t.Errorf("expected '%s' but got '%s'", test.expected, res)
}
logOutput := strings.TrimSuffix(buf.String(), "\n")
if !strings.Contains(logOutput, "markdown options is not valid: ") {
t.Errorf("expected a different log message but got '%s'", logOutput)
}
})
}
}
func BenchmarkFromString(b *testing.B) {
converter := NewConverter("www.google.com", true, nil)
strongRule := Rule{
Filter: []string{"strong"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
return nil
},
}
var wg sync.WaitGroup
convert := func(html string) {
defer wg.Done()
_, err := converter.ConvertString(html)
if err != nil {
b.Error(err)
}
}
add := func() {
defer wg.Done()
converter.AddRules(strongRule)
}
for n := 0; n < b.N; n++ {
wg.Add(2)
go add()
go convert("Bold")
}
wg.Wait()
}
func TestAddRules_ChangeContent(t *testing.T) {
expected := "Some other Content"
var wasCalled bool
rule := Rule{
Filter: []string{"p"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
wasCalled = true
if content != "Some Content" {
t.Errorf("got wrong `content`: '%s'", content)
}
if !selec.Is("p") {
t.Error("selec is not p")
}
return String(expected)
},
}
conv := NewConverter("", true, nil)
conv.AddRules(rule)
md, err := conv.ConvertString(`
Some Content
`)
if err != nil {
t.Error(err)
}
if md != expected {
t.Errorf("wanted '%s' but got '%s'", expected, md)
}
if !wasCalled {
t.Error("rule was not called")
}
}
func TestAddRules_Fallback(t *testing.T) {
// firstExpected := "Some other Content"
expected := "Totally different Content"
var firstWasCalled bool
var secondWasCalled bool
firstRule := Rule{
Filter: []string{"p"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
firstWasCalled = true
if secondWasCalled {
t.Error("expected first rule to be called before second rule. second is already called")
}
if content != "Some Content" {
t.Errorf("got wrong `content`: '%s'", content)
}
if !selec.Is("p") {
t.Error("selec is not p")
}
return nil
},
}
secondRule := Rule{
Filter: []string{"p"},
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
secondWasCalled = true
if !firstWasCalled {
t.Error("expected first rule to be called before second rule. first is not called yet")
}
if content != "Some Content" {
t.Errorf("got wrong `content`: '%s'", content)
}
if !selec.Is("p") {
t.Error("selec is not p")
}
return String(expected)
},
}
conv := NewConverter("", true, nil)
conv.AddRules(secondRule, firstRule)
md, err := conv.ConvertString(`
Some Content
`)
if err != nil {
t.Error(err)
}
if md != expected {
t.Errorf("wanted '%s' but got '%s'", expected, md)
}
if !firstWasCalled {
t.Error("first rule was not called")
}
if !secondWasCalled {
t.Error("second rule was not called")
}
}
func TestAddRules_NoRules(t *testing.T) {
var buf bytes.Buffer
log.SetOutput(&buf)
log.SetFlags(0)
defer func() {
// reset the options back to the defaults
log.SetOutput(os.Stderr)
log.SetFlags(3)
}()
var wasCalled bool
rule := Rule{
Filter: []string{ /* nothing */ },
Replacement: func(content string, selec *goquery.Selection, opt *Options) *string {
wasCalled = true
return nil
},
}
conv := NewConverter("", true, nil)
conv.AddRules(rule)
md, err := conv.ConvertString(`
Text und Fotos: Max Mustermann Original veröffentlicht am 01.01.2021 auf Zeitung Übersetzung: Max Mustermann veröffentlicht am 02.02.2021
output.asterisks.golden 0000664 0000000 0000000 00000000515 14624642056 0035510 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/italic Some *Text*
Some *Text*
Some *Text*
*DoubleItalic*
Some *19,80€* Text
*Content* and no space afterward.
\_Not Italic\_
*First*
*Second*
*Text und Fotos: Max Mustermann*
*Original veröffentlicht am 01.01.2021 auf* [Zeitung](http://example.com/posts/100979)
*Übersetzung: Max Mustermann*
*veröffentlicht am 02.02.2021* output.underscores.golden 0000664 0000000 0000000 00000000515 14624642056 0036034 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/italic Some _Text_
Some _Text_
Some _Text_
_DoubleItalic_
Some _19,80€_ Text
_Content_ and no space afterward.
\_Not Italic\_
_First_
_Second_
_Text und Fotos: Max Mustermann_
_Original veröffentlicht am 01.01.2021 auf_ [Zeitung](http://example.com/posts/100979)
_Übersetzung: Max Mustermann_
_veröffentlicht am 02.02.2021_ golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/keep_remove_tag/ 0000775 0000000 0000000 00000000000 14624642056 0032714 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000000163 14624642056 0035627 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/keep_remove_tag
golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link/ 0000775 0000000 0000000 00000000000 14624642056 0030515 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link/input.html 0000664 0000000 0000000 00000004651 14624642056 0032550 0 ustar 00root root 0000000 0000000
Simple Absolute Link Simple Relative Link Link with Space Link with Title Link with multiline Title Broken Link
output.inlined.golden 0000664 0000000 0000000 00000002375 14624642056 0034620 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link [Simple Absolute Link](http://simple.org/)
[Simple Relative Link](http://example.com/page.html)
[Link with Space](http://space.org/)
[Link with Title](http://title.org/ "Some Title")
[Link with multiline Title](http://title.org/ "Some Long Title")
[Broken Link](http://broken.com/test.html%5Cr)
[First Text\
\
Second Text](http://multi.org/)
- [First Text\
\
Second Text](http://list.org/)
[GitHub](https://github.com "GitHub")
[first top](http://first_under.com) [second below](http://second_under.com)
[first left](http://first_next.com) [second right](http://second_next.com)
Before [close](http://example_close.com) After
[**Heading A** **Heading B**](http://example.com/page.html)
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"](http://example.com/page.html "\"Vermögenssteuer ist aus wirtschaftlicher Sicht klug\"")
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**](http://example.com/nachrichten/wdr-aktuell-app-stores-100.html "Die App WDR aktuell begleitet Sie durch den Tag ") output.referenced_collapsed.golden 0000664 0000000 0000000 00000003653 14624642056 0037326 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link [Simple Absolute Link][]
[Simple Relative Link][]
[Link with Space][]
[Link with Title][]
[Link with multiline Title][]
[Broken Link][]
[First Text\
\
Second Text][]
- [First Text\
\
Second Text][]
[GitHub][]
[first top][] [second below][]
[first left][] [second right][]
Before [close][] After
[**Heading A** **Heading B**][]
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"][]
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**][]
[Simple Absolute Link]: http://simple.org/
[Simple Relative Link]: http://example.com/page.html
[Link with Space]: http://space.org/
[Link with Title]: http://title.org/ "Some Title"
[Link with multiline Title]: http://title.org/ "Some Long Title"
[Broken Link]: http://broken.com/test.html%5Cr
[First Text\
\
Second Text]: http://multi.org/
[First Text\
\
Second Text]: http://list.org/
[GitHub]: https://github.com "GitHub"
[first top]: http://first_under.com
[second below]: http://second_under.com
[first left]: http://first_next.com
[second right]: http://second_next.com
[close]: http://example_close.com
[**Heading A** **Heading B**]: http://example.com/page.html
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"]: http://example.com/page.html "\"Vermögenssteuer ist aus wirtschaftlicher Sicht klug\""
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**]: http://example.com/nachrichten/wdr-aktuell-app-stores-100.html "Die App WDR aktuell begleitet Sie durch den Tag " output.referenced_full.golden 0000664 0000000 0000000 00000002605 14624642056 0036316 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link [Simple Absolute Link][1]
[Simple Relative Link][2]
[Link with Space][3]
[Link with Title][4]
[Link with multiline Title][5]
[Broken Link][6]
[First Text\
\
Second Text][7]
- [First Text\
\
Second Text][8]
[GitHub][9]
[first top][10] [second below][11]
[first left][12] [second right][13]
Before [close][14] After
[**Heading A** **Heading B**][15]
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"][16]
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**][17]
[1]: http://simple.org/
[2]: http://example.com/page.html
[3]: http://space.org/
[4]: http://title.org/ "Some Title"
[5]: http://title.org/ "Some Long Title"
[6]: http://broken.com/test.html%5Cr
[7]: http://multi.org/
[8]: http://list.org/
[9]: https://github.com "GitHub"
[10]: http://first_under.com
[11]: http://second_under.com
[12]: http://first_next.com
[13]: http://second_next.com
[14]: http://example_close.com
[15]: http://example.com/page.html
[16]: http://example.com/page.html "\"Vermögenssteuer ist aus wirtschaftlicher Sicht klug\""
[17]: http://example.com/nachrichten/wdr-aktuell-app-stores-100.html "Die App WDR aktuell begleitet Sie durch den Tag " output.referenced_shortcut.golden 0000664 0000000 0000000 00000003611 14624642056 0037225 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link [Simple Absolute Link]
[Simple Relative Link]
[Link with Space]
[Link with Title]
[Link with multiline Title]
[Broken Link]
[First Text\
\
Second Text]
- [First Text\
\
Second Text]
[GitHub]
[first top] [second below]
[first left] [second right]
Before [close] After
[**Heading A** **Heading B**]
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"]
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**]
[Simple Absolute Link]: http://simple.org/
[Simple Relative Link]: http://example.com/page.html
[Link with Space]: http://space.org/
[Link with Title]: http://title.org/ "Some Title"
[Link with multiline Title]: http://title.org/ "Some Long Title"
[Broken Link]: http://broken.com/test.html%5Cr
[First Text\
\
Second Text]: http://multi.org/
[First Text\
\
Second Text]: http://list.org/
[GitHub]: https://github.com "GitHub"
[first top]: http://first_under.com
[second below]: http://second_under.com
[first left]: http://first_next.com
[second right]: http://second_next.com
[close]: http://example_close.com
[**Heading A** **Heading B**]: http://example.com/page.html
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"]: http://example.com/page.html "\"Vermögenssteuer ist aus wirtschaftlicher Sicht klug\""
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**]: http://example.com/nachrichten/wdr-aktuell-app-stores-100.html "Die App WDR aktuell begleitet Sie durch den Tag " output.relative.golden 0000664 0000000 0000000 00000002240 14624642056 0035000 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/link [Simple Absolute Link](http://simple.org/)
[Simple Relative Link](/page.html)
[Link with Space](http://space.org/)
[Link with Title](http://title.org/ "Some Title")
[Link with multiline Title](http://title.org/ "Some Long Title")
[Broken Link](http://broken.com/test.html\r)
[First Text\
\
Second Text](http://multi.org/)
- [First Text\
\
Second Text](http://list.org/)
[GitHub](https://github.com "GitHub")
[first top](http://first_under.com) [second below](http://second_under.com)
[first left](http://first_next.com) [second right](http://second_next.com)
Before [close](http://example_close.com) After
[**Heading A** **Heading B**](/page.html)
[DIW-Chef zum Grünen-Programm "Vermögenssteuer ist aus wirtschaftlicher Sicht klug"](/page.html "\"Vermögenssteuer ist aus wirtschaftlicher Sicht klug\"")
[**Die App WDR aktuell begleitet Sie durch den Tag**\
\
Sie möchten eine App, die Sie so durch den Tag in NRW begleitet, dass Sie jederzeit mitreden können? Die App WDR aktuell bietet Ihnen dafür immer die passenden Nachrichten.\
\| \
**mehr**](/nachrichten/wdr-aktuell-app-stores-100.html "Die App WDR aktuell begleitet Sie durch den Tag ") golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list/ 0000775 0000000 0000000 00000000000 14624642056 0030533 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list/goldmark.golden 0000664 0000000 0000000 00000005257 14624642056 0033536 0 ustar 00root root 0000000 0000000
All manually reviewed Drosophila melanogaster entries
All manually reviewed Drosophila pseudoobscura pseudoobscura entries
before middle after
beforemiddleafter
beforemiddle after
before middleafter
Not in a list
output.asterisks.golden 0000664 0000000 0000000 00000002617 14624642056 0035223 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list * Some Thing
* Another Thing
1. First Thing
2. Second Thing
01. 1
02. 2
03. 3
04. 4
05. 5
06. 6
07. 7
08. 8
09. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
19. 16
20. 17
21. 18
22. 19
23. 20
24. 
25. 22
10. 10
11. 11
12. 12
13. 13
1. 1
2. 2
3. 3
4. 4
1. 1
2. 2
3. 3
4. 4
0. 0
1. 1
2. 2
3. 3
3. Echo the word "foo"
```
echo('foo')
```
4. Now echo "bar"
```
echo('bar')
```
* Link: [example](https://example.com) works
* Link:
[example](https://example.com)
works
* Link:
[example](https://example.com)
works
* Link:
[example](https://example.com)
works
* Link:
[example](https://example.com)
works
1. Link: [example](https://example.com) works
2. Link:
[example](https://example.com)
works
1. First Thing
* Some Thing
* Another Thing
2. Second Thing
* foo
* bar
* List items
* Ending with
* A space
* Indent First Thing
Second Thing
* Third Thing
\- Not List
1\. Not List 1. Not List
1\. Not List
1. A paragraph
with two lines.
```
indented code
```
> A block quote.
* [](http://example.com/icon)
### Title
Description
* All manually reviewed _Drosophila melanogaster_ entries
* All manually reviewed _Drosophila pseudoobscura pseudoobscura_ entries
* before _middle_ after
* before _middle_ after
* before _middle_ after
* before _middle_ after
Not in a list output.dash.golden 0000664 0000000 0000000 00000002617 14624642056 0034132 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list - Some Thing
- Another Thing
1. First Thing
2. Second Thing
01. 1
02. 2
03. 3
04. 4
05. 5
06. 6
07. 7
08. 8
09. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
19. 16
20. 17
21. 18
22. 19
23. 20
24. 
25. 22
10. 10
11. 11
12. 12
13. 13
1. 1
2. 2
3. 3
4. 4
1. 1
2. 2
3. 3
4. 4
0. 0
1. 1
2. 2
3. 3
3. Echo the word "foo"
```
echo('foo')
```
4. Now echo "bar"
```
echo('bar')
```
- Link: [example](https://example.com) works
- Link:
[example](https://example.com)
works
- Link:
[example](https://example.com)
works
- Link:
[example](https://example.com)
works
- Link:
[example](https://example.com)
works
1. Link: [example](https://example.com) works
2. Link:
[example](https://example.com)
works
1. First Thing
- Some Thing
- Another Thing
2. Second Thing
- foo
- bar
- List items
- Ending with
- A space
- Indent First Thing
Second Thing
- Third Thing
\- Not List
1\. Not List 1. Not List
1\. Not List
1. A paragraph
with two lines.
```
indented code
```
> A block quote.
- [](http://example.com/icon)
### Title
Description
- All manually reviewed _Drosophila melanogaster_ entries
- All manually reviewed _Drosophila pseudoobscura pseudoobscura_ entries
- before _middle_ after
- before _middle_ after
- before _middle_ after
- before _middle_ after
Not in a list output.plus.golden 0000664 0000000 0000000 00000002617 14624642056 0034176 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list + Some Thing
+ Another Thing
1. First Thing
2. Second Thing
01. 1
02. 2
03. 3
04. 4
05. 5
06. 6
07. 7
08. 8
09. 9
10. 10
11. 11
12. 12
13. 13
14. 14
15. 15
19. 16
20. 17
21. 18
22. 19
23. 20
24. 
25. 22
10. 10
11. 11
12. 12
13. 13
1. 1
2. 2
3. 3
4. 4
1. 1
2. 2
3. 3
4. 4
0. 0
1. 1
2. 2
3. 3
3. Echo the word "foo"
```
echo('foo')
```
4. Now echo "bar"
```
echo('bar')
```
+ Link: [example](https://example.com) works
+ Link:
[example](https://example.com)
works
+ Link:
[example](https://example.com)
works
+ Link:
[example](https://example.com)
works
+ Link:
[example](https://example.com)
works
1. Link: [example](https://example.com) works
2. Link:
[example](https://example.com)
works
1. First Thing
+ Some Thing
+ Another Thing
2. Second Thing
+ foo
+ bar
+ List items
+ Ending with
+ A space
+ Indent First Thing
Second Thing
+ Third Thing
\- Not List
1\. Not List 1. Not List
1\. Not List
1. A paragraph
with two lines.
```
indented code
```
> A block quote.
+ [](http://example.com/icon)
### Title
Description
+ All manually reviewed _Drosophila melanogaster_ entries
+ All manually reviewed _Drosophila pseudoobscura pseudoobscura_ entries
+ before _middle_ after
+ before _middle_ after
+ before _middle_ after
+ before _middle_ after
Not in a list golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list_nested/ 0000775 0000000 0000000 00000000000 14624642056 0032075 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list_nested/input.html0000664 0000000 0000000 00000007375 14624642056 0034136 0 ustar 00root root 0000000 0000000
The <img> tag is used to embed an image.
The tag is used to embed an image.
Does it work?
The Corinthos Center for Cancer will be partially closed for remodeling
starting 4/15/21. Patients should be redirected as space
permits in the following order:
Metro Court West.
Richie General.
This place is ok.
Watch out for the doctors.
They bite.
But not hard.
Port Charles Main.
For further information about appointment changes, contact:
Dorothy Hardy
Head of Operations
Interim
dorothy.hardy@generalhospital.org
555-555-5555
The remodel is expected to complete in June 2021.Timeframe subject to change.
output.asterisks.golden 0000664 0000000 0000000 00000003262 14624642056 0036562 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list_nested * foo
* bar
* baz
* boo
* Coffee
* Tea
* Black tea
* Green tea
* Milk
# header1
* Bullet list
* Nested bullet
* Sub-nested bullet etc
* Bullet list item 2
* One
* One point one
* One point two
1. One
1. One point one
2. One point two
* 1
* 2
* 2.1
* 2.2
* 2.2.1
* 2.2.2
* 2.2.3
* 3
1. 1
2. 2
1. 2.1
2. 2.2
1. 2.2.1
2. 2.2.2
3. 2.2.3
4. 3
* First Thing
Second Thing
* Nested First Thing
Nested Thing
1. First Thing
Second Thing
1. Nested First Thing
Nested Thing
2. Date:
20.02.2021
26\. Mai - 3. Juni
* [Gesellschaft](http://example.com/gesellschaft/ "Gesellschaft") * [Panorama](http://example.com/gesellschaft/panorama/ "Panorama")
* [Medien](http://example.com/gesellschaft/medien/ "Medien")
* [Geschichte](http://example.com/gesellschaft/geschichte/ "Geschichte")
* What...
* about...
##### different elements:
`variable`
* * *
```
The tag is used to embed an image.
The tag is used to embed an image.
```
Does it work?
The Corinthos Center for Cancer will be partially closed for remodeling
starting **4/15/21**. Patients should be redirected as space
permits in the following order:
1. Metro Court West.
2. Richie General.
1. This place is ok.
2. Watch out for the doctors.
1. They bite.
2. But not hard.
4. Port Charles Main.
For further information about appointment changes, contact:
* Dorothy Hardy
* _Head of Operations_
* _Interim_
* dorothy.hardy@generalhospital.org
* 555-555-5555
_The remodel is_ [_expected_](http://www.google.com/) _to complete in June 2021._ **_Timeframe subject to change_** _._ output.dash.golden 0000664 0000000 0000000 00000003262 14624642056 0035471 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list_nested - foo
- bar
- baz
- boo
- Coffee
- Tea
- Black tea
- Green tea
- Milk
# header1
- Bullet list
- Nested bullet
- Sub-nested bullet etc
- Bullet list item 2
- One
- One point one
- One point two
1. One
1. One point one
2. One point two
- 1
- 2
- 2.1
- 2.2
- 2.2.1
- 2.2.2
- 2.2.3
- 3
1. 1
2. 2
1. 2.1
2. 2.2
1. 2.2.1
2. 2.2.2
3. 2.2.3
4. 3
- First Thing
Second Thing
- Nested First Thing
Nested Thing
1. First Thing
Second Thing
1. Nested First Thing
Nested Thing
2. Date:
20.02.2021
26\. Mai - 3. Juni
- [Gesellschaft](http://example.com/gesellschaft/ "Gesellschaft") - [Panorama](http://example.com/gesellschaft/panorama/ "Panorama")
- [Medien](http://example.com/gesellschaft/medien/ "Medien")
- [Geschichte](http://example.com/gesellschaft/geschichte/ "Geschichte")
- What...
- about...
##### different elements:
`variable`
* * *
```
The tag is used to embed an image.
The tag is used to embed an image.
```
Does it work?
The Corinthos Center for Cancer will be partially closed for remodeling
starting **4/15/21**. Patients should be redirected as space
permits in the following order:
1. Metro Court West.
2. Richie General.
1. This place is ok.
2. Watch out for the doctors.
1. They bite.
2. But not hard.
4. Port Charles Main.
For further information about appointment changes, contact:
- Dorothy Hardy
- _Head of Operations_
- _Interim_
- dorothy.hardy@generalhospital.org
- 555-555-5555
_The remodel is_ [_expected_](http://www.google.com/) _to complete in June 2021._ **_Timeframe subject to change_** _._ output.plus.golden 0000664 0000000 0000000 00000003262 14624642056 0035535 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/list_nested + foo
+ bar
+ baz
+ boo
+ Coffee
+ Tea
+ Black tea
+ Green tea
+ Milk
# header1
+ Bullet list
+ Nested bullet
+ Sub-nested bullet etc
+ Bullet list item 2
+ One
+ One point one
+ One point two
1. One
1. One point one
2. One point two
+ 1
+ 2
+ 2.1
+ 2.2
+ 2.2.1
+ 2.2.2
+ 2.2.3
+ 3
1. 1
2. 2
1. 2.1
2. 2.2
1. 2.2.1
2. 2.2.2
3. 2.2.3
4. 3
+ First Thing
Second Thing
+ Nested First Thing
Nested Thing
1. First Thing
Second Thing
1. Nested First Thing
Nested Thing
2. Date:
20.02.2021
26\. Mai - 3. Juni
+ [Gesellschaft](http://example.com/gesellschaft/ "Gesellschaft") + [Panorama](http://example.com/gesellschaft/panorama/ "Panorama")
+ [Medien](http://example.com/gesellschaft/medien/ "Medien")
+ [Geschichte](http://example.com/gesellschaft/geschichte/ "Geschichte")
+ What...
+ about...
##### different elements:
`variable`
* * *
```
The tag is used to embed an image.
The tag is used to embed an image.
```
Does it work?
The Corinthos Center for Cancer will be partially closed for remodeling
starting **4/15/21**. Patients should be redirected as space
permits in the following order:
1. Metro Court West.
2. Richie General.
1. This place is ok.
2. Watch out for the doctors.
1. They bite.
2. But not hard.
4. Port Charles Main.
For further information about appointment changes, contact:
+ Dorothy Hardy
+ _Head of Operations_
+ _Interim_
+ dorothy.hardy@generalhospital.org
+ 555-555-5555
_The remodel is_ [_expected_](http://www.google.com/) _to complete in June 2021._ **_Timeframe subject to change_** _._ golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/p_tag/ 0000775 0000000 0000000 00000000000 14624642056 0030652 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/p_tag/goldmark.golden 0000664 0000000 0000000 00000001112 14624642056 0033637 0 ustar 00root root 0000000 0000000
Some Content
Text
Some Text
Some Content
jmap –histo[:live]
Sometimes a struct field, function, type, or even a whole package becomes
redundant or unnecessary, but must be kept for compatibility with existing
programs.
To signal that an identifier should not be used, add a paragraph to its doc
comment that begins with "Deprecated:" followed by some information about the
Sometimes a struct field, function, type, or even a whole package becomes
redundant or unnecessary, but must be kept for compatibility with existing
programs.
To signal that an identifier should not be used, add a paragraph to its doc
comment that begins with "Deprecated:" followed by some information about the
deprecation.
There are a few examples in the standard library.
output.default.golden 0000664 0000000 0000000 00000000755 14624642056 0034757 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/p_tag Some Content
Text
Some Text
Some Content
jmap –histo\[:live\]
Sometimes a struct field, function, type, or even a whole package becomes
redundant or unnecessary, but must be kept for compatibility with existing
programs.
To signal that an identifier should not be used, add a paragraph to its doc
comment that begins with "Deprecated:" followed by some information about the
deprecation.
There are a few examples [in the standard library](https://golang.org/search?q=Deprecated:). golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code/ 0000775 0000000 0000000 00000000000 14624642056 0031340 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000006554 14624642056 0034265 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code
last_30_days
with backtick (`)
with backtick (``)
here are three ``` here are four ```` here is one ` that's it
`starting & ending with a backtick`
Who ate the most donuts this week?
Jeff 15
Sam 11
Robin 6
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
When x = 3, that means x + 2 = 5
The <img> tag is used to embed an image.
The `` tag is used to embed an image.
Two variables AB
CSS: body { color: yellow; font-size: 16px; }
CSS: body { color: yellow; font-size: 16px; }
```
~~~
Some ~~~
totally ~~~~~~ normal
~ code
The <img> tag is used to embed an image.
The tag is used to embed an image.
function fn() {
x = 1;
return x;
// eslint-disable-next-line no-unreachable
x = 3;
}
<p><span>Some text</span></p>
Copy to Clipboard
HTML:
<p>Using CSS to change the font color is easy.</p>
<pre>
body {
color: red;
}
</pre>
Copy to Clipboard
---
title: "Hello! This is the markdown file"
date: 2021-09-25
tags: ["react"]
---
Content of the post goes here.
<code class="language-css">p { color: red }</code>
The function selectAll() highlights all the text in the
input field so the user can, for example, copy or delete the text.
The <pre>HTML element represents preformatted text.
This is the content:
_code_
_pre_
Cool!
A simple equation: x = y + 2
When the process is complete, the utility will output the text
Scan complete. Found N results. You can then
proceed to the next step.
Use the command help mycommand to view documentation
for the command "mycommand".
The telnet client should display: Local Echo is on
The operator (#) takes a bit x and a
sequence a and produces a new sequence x # a with
x as the head and a as the tail (very much like the
built-in operation (:) for lists):
golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code/input.html 0000664 0000000 0000000 00000017477 14624642056 0033405 0 ustar 00root root 0000000 0000000
last_30_days with backtick (`) with backtick (``) here are three ``` here are four ```` here is one ` that's it `starting & ending with a backtick`
Who ate the most donuts this week?
Jeff 15
Sam 11
Robin 6
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
When x = 3, that means x + 2 = 5
The <img> tag is used to embed an image.
The tag is used to embed an image.
Two variables AB
CSS:
body {
color: yellow;
font-size: 16px;
}
CSS:
body {
color: yellow;
font-size: 16px;
}
```
~~~
Some ~~~
totally ~~~~~~ normal
~ code
The <img> tag is used to embed an image.
The tag is used to embed an image.
functionfn(){
x =1;return x;// eslint-disable-next-line no-unreachable
x =3;}
<p><span>Some text</span></p>
HTML:
<p>Using CSS to change the font color is easy.</p><pre>
body {
color: red;
}
</pre>
---title: "Hello! This is the markdown file"
date: 2021-09-25
tags: ["react"]---
Content of the post goes here.
<codeclass="language-css">p { color: red }</code>
The function selectAll() highlights all the text in the
input field so the user can, for example, copy or delete the text.
The <pre>HTML element represents preformatted text.
This is the content:
_code_
_pre_
Cool!
A simple equation: x = y + 2
When the process is complete, the utility will output the text
Scan complete. Found N results. You can then
proceed to the next step.
Use the command help mycommand to view documentation
for the command "mycommand".
The telnet client should display: Local Echo is on
The operator (#) takes a bit x and a
sequence a and produces a new sequence x # a with
x as the head and a as the tail (very much like the
built-in operation (:) for lists):
output.fenced_backtick.golden 0000664 0000000 0000000 00000004774 14624642056 0037105 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code `last_30_days`
``with backtick (`)``
```with backtick (``)```
`````here are three ``` here are four ```` here is one ` that's it`````
`` `starting & ending with a backtick` ``
Who ate the most donuts this week?
```foo+bar
Jeff 15
Sam 11
Robin 6
```
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
When `x = 3`, that means `x + 2 = 5`
The `` tag is used to embed an image.
The `` tag is used to embed an image.
Two variables `A` `B`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
````
```
````
```
~~~
```
```
Some ~~~
totally ~~~~~~ normal
~ code
```
```
The tag is used to embed an image.
The tag is used to embed an image.
```
```
One
Two
Three
```
```
```
We can do this like so:
```js
window.location.pathname.split("/");
// ["", "blog", "javascript", "how-to-get-the-last-segment-of-a-url-in-javascript", ""]
```
* * *
```js
function fn() {
x = 1;
return x;
// eslint-disable-next-line no-unreachable
x = 3;
}
```
* * *
```
Some text
```
Copy to Clipboard
* * *
#### HTML:
```
Using CSS to change the font color is easy.
body {
color: red;
}
```
Copy to Clipboard
* * *
```md
---
title: "Hello! This is the markdown file"
date: 2021-09-25
tags: ["react"]
---
Content of the post goes here.
```
* * *
```markup
p { color: red }
```
* * *
The function `selectAll()` highlights all the text in the
input field so the user can, for example, copy or delete the text.
The **`
`** [HTML](http://example.com/en-US/docs/Web/HTML) element represents preformatted text.
* * *
```
This is the content:
_code_
_pre_
Cool!
```
* * *
A simple equation: x = y \+ 2
When the process is complete, the utility will output the text
`Scan complete. Found N results.` You can then
proceed to the next step.
Use the command `help mycommand` to view documentation
for the command "mycommand".
The telnet client should display: `Local Echo is on`
The operator `(#)` takes a bit `x` and a
sequence `a` and produces a new sequence `x # a` with
`x` as the head and `a` as the tail (very much like the
built-in operation `(:)` for lists): output.fenced_tilde.golden 0000664 0000000 0000000 00000005004 14624642056 0036416 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code `last_30_days`
``with backtick (`)``
```with backtick (``)```
`````here are three ``` here are four ```` here is one ` that's it`````
`` `starting & ending with a backtick` ``
Who ate the most donuts this week?
~~~foo+bar
Jeff 15
Sam 11
Robin 6
~~~
~~~
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
~~~
When `x = 3`, that means `x + 2 = 5`
The `` tag is used to embed an image.
The `` tag is used to embed an image.
Two variables `A` `B`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
~~~
```
~~~
~~~~
~~~
~~~~
~~~~~~~
Some ~~~
totally ~~~~~~ normal
~ code
~~~~~~~
~~~
The tag is used to embed an image.
The tag is used to embed an image.
~~~
~~~
One
Two
Three
~~~
~~~
~~~
We can do this like so:
~~~js
window.location.pathname.split("/");
// ["", "blog", "javascript", "how-to-get-the-last-segment-of-a-url-in-javascript", ""]
~~~
* * *
~~~js
function fn() {
x = 1;
return x;
// eslint-disable-next-line no-unreachable
x = 3;
}
~~~
* * *
~~~
Some text
~~~
Copy to Clipboard
* * *
#### HTML:
~~~
Using CSS to change the font color is easy.
body {
color: red;
}
~~~
Copy to Clipboard
* * *
~~~md
---
title: "Hello! This is the markdown file"
date: 2021-09-25
tags: ["react"]
---
Content of the post goes here.
~~~
* * *
~~~markup
p { color: red }
~~~
* * *
The function `selectAll()` highlights all the text in the
input field so the user can, for example, copy or delete the text.
The **`
`** [HTML](http://example.com/en-US/docs/Web/HTML) element represents preformatted text.
* * *
~~~
This is the content:
_code_
_pre_
Cool!
~~~
* * *
A simple equation: x = y \+ 2
When the process is complete, the utility will output the text
`Scan complete. Found N results.` You can then
proceed to the next step.
Use the command `help mycommand` to view documentation
for the command "mycommand".
The telnet client should display: `Local Echo is on`
The operator `(#)` takes a bit `x` and a
sequence `a` and produces a new sequence `x # a` with
`x` as the head and `a` as the tail (very much like the
built-in operation `(:)` for lists): output.indented.golden 0000664 0000000 0000000 00000004774 14624642056 0035620 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/pre_code `last_30_days`
``with backtick (`)``
```with backtick (``)```
`````here are three ``` here are four ```` here is one ` that's it`````
`` `starting & ending with a backtick` ``
Who ate the most donuts this week?
```foo+bar
Jeff 15
Sam 11
Robin 6
```
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
When `x = 3`, that means `x + 2 = 5`
The `` tag is used to embed an image.
The `` tag is used to embed an image.
Two variables `A` `B`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
CSS: `
body {
color: yellow;
font-size: 16px;
}
`
````
```
````
```
~~~
```
```
Some ~~~
totally ~~~~~~ normal
~ code
```
```
The tag is used to embed an image.
The tag is used to embed an image.
```
```
One
Two
Three
```
```
```
We can do this like so:
```js
window.location.pathname.split("/");
// ["", "blog", "javascript", "how-to-get-the-last-segment-of-a-url-in-javascript", ""]
```
* * *
```js
function fn() {
x = 1;
return x;
// eslint-disable-next-line no-unreachable
x = 3;
}
```
* * *
```
Some text
```
Copy to Clipboard
* * *
#### HTML:
```
Using CSS to change the font color is easy.
body {
color: red;
}
```
Copy to Clipboard
* * *
```md
---
title: "Hello! This is the markdown file"
date: 2021-09-25
tags: ["react"]
---
Content of the post goes here.
```
* * *
```markup
p { color: red }
```
* * *
The function `selectAll()` highlights all the text in the
input field so the user can, for example, copy or delete the text.
The **`
`** [HTML](http://example.com/en-US/docs/Web/HTML) element represents preformatted text.
* * *
```
This is the content:
_code_
_pre_
Cool!
```
* * *
A simple equation: x = y \+ 2
When the process is complete, the utility will output the text
`Scan complete. Found N results.` You can then
proceed to the next step.
Use the command `help mycommand` to view documentation
for the command "mycommand".
The telnet client should display: `Local Echo is on`
The operator `(#)` takes a bit `x` and a
sequence `a` and produces a new sequence `x # a` with
`x` as the head and `a` as the tail (very much like the
built-in operation `(:)` for lists): golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/sup_element/ 0000775 0000000 0000000 00000000000 14624642056 0032100 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000000334 14624642056 0035013 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/sup_element
One of the most common equations in all of physics is
E=mc2.
The ordinal number "fifth" can be abbreviated in various languages as follows:
One of the most common equations in all of physics is
E=mc2.
The ordinal number "fifth" can be abbreviated in various languages as follows:
English: 5th
French: 5ème
output.default.golden 0000664 0000000 0000000 00000000254 14624642056 0036177 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/sup_element One of the most common equations in all of physics is
E=mc2.
The ordinal number "fifth" can be abbreviated in various languages as follows:
- English: 5th
- French: 5ème golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestPlugins/ 0000775 0000000 0000000 00000000000 14624642056 0027076 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestPlugins/checkbox/ 0000775 0000000 0000000 00000000000 14624642056 0030664 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestPlugins/checkbox/goldmark.golden 0000664 0000000 0000000 00000001136 14624642056 0033657 0 ustar 00root root 0000000 0000000
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
godoc documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
documentation to its
implementation with one click.
Godoc is conceptually related to Python's
Docstring and Java's
Javadoc, but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
fmt package's
Fprint function:
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
sort package's brief description:
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
They can also be detailed like the
gob package's overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
doc.go, which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
package list.
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
"BUG(who)” are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
bytes package:
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
in the standard library.
There are a few formatting rules that Godoc uses when converting comments to HTML:
Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
Pre-formatted text must be indented relative to the surrounding comment text (see gob's
doc.go for an example).
URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
$GOROOT/src/pkg and any
GOPATH work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
-path flag or just by running
"godoc ." in the source directory. See the
godoc documentation for more details.
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
godoc documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
documentation to its
implementation with one click.
Godoc is conceptually related to Python's
Docstring and Java's
Javadoc, but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
fmt package's
Fprint function:
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
sort package's brief description:
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
They can also be detailed like the
gob package's overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
doc.go, which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
package list.
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
"BUG(who)” are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
bytes package:
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
in the standard library.
There are a few formatting rules that Godoc uses when converting comments to HTML:
Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
Pre-formatted text must be indented relative to the surrounding comment text (see gob's
doc.go for an example).
URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
$GOROOT/src/pkg and any
GOPATH work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
-path flag or just by running
"godoc ." in the source directory. See the
godoc documentation for more details.
output.emphasis_asterisks.golden 0000664 0000000 0000000 00000021373 14624642056 0040710 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language](http://golang.org/)
[Go](http://golang.org/)
▽
[Documents](http://golang.org/doc/) [Packages](http://golang.org/pkg/) [The Project](http://golang.org/project/) [Help](http://golang.org/help/) [Blog](http://blog.golang.org/)submit search
#### Next article
[Introducing Gofix](http://blog.golang.org/introducing-gofix)
#### Previous article
[Gobs of data](http://blog.golang.org/gobs-of-data)
#### Links
- [golang.org](http://golang.org/)
- [Install Go](http://golang.org/doc/install.html)
- [A Tour of Go](http://tour.golang.org/)
- [Go Documentation](http://golang.org/doc/)
- [Go Mailing List](http://groups.google.com/group/golang-nuts)
- [Go on Google+](http://plus.google.com/101406623878176903605)
- [Go+ Community](http://plus.google.com/communities/114112804251407510571)
- [Go on Twitter](http://twitter.com/golang)
[Blog index](http://blog.golang.org/index)
# [The Go Blog](http://blog.golang.org/)
### [Godoc: documenting Go code](http://blog.golang.org/godoc-documenting-go-code)
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc](https://golang.org/cmd/godoc/) documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation](https://golang.org/pkg/strings/#HasPrefix) to its
[implementation](https://golang.org/src/pkg/strings/strings.go#L493) with one click.
Godoc is conceptually related to Python's
[Docstring](http://www.python.org/dev/peps/pep-0257/) and Java's
[Javadoc](http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html), but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`](https://golang.org/pkg/fmt/#Fprint) function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`](https://golang.org/pkg/sort/) package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package](https://golang.org/pkg/encoding/gob/)'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go), which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list](https://golang.org/pkg/).
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package](https://golang.org/pkg/bytes/#pkg-note-BUG):
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library](https://golang.org/search?q=Deprecated:).
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go) for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation](https://golang.org/cmd/godoc/) for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push](http://blog.golang.org/h2push)
- [Introducing HTTP Tracing](http://blog.golang.org/http-tracing)
- [Testable Examples in Go](http://blog.golang.org/examples)
- [Generating code](http://blog.golang.org/generate)
- [Introducing the Go Race Detector](http://blog.golang.org/race-detector)
- [Go maps in action](http://blog.golang.org/go-maps-in-action)
- [go fmt your code](http://blog.golang.org/go-fmt-your-code)
- [Organizing Go code](http://blog.golang.org/organizing-go-code)
- [Debugging Go programs with the GNU Debugger](http://blog.golang.org/debugging-go-programs-with-gnu-debugger)
- [The Go image/draw package](http://blog.golang.org/go-imagedraw-package)
- [The Go image package](http://blog.golang.org/go-image-package)
- [The Laws of Reflection](http://blog.golang.org/laws-of-reflection)
- [Error handling and Go](http://blog.golang.org/error-handling-and-go)
- ["First Class Functions in Go"](http://blog.golang.org/first-class-functions-in-go-and-new-go)
- [Profiling Go Programs](http://blog.golang.org/profiling-go-programs)
- [A GIF decoder: an exercise in Go interfaces](http://blog.golang.org/gif-decoder-exercise-in-go-interfaces)
- [Introducing Gofix](http://blog.golang.org/introducing-gofix)
- [Gobs of data](http://blog.golang.org/gobs-of-data)
- [C? Go? Cgo!](http://blog.golang.org/c-go-cgo)
- [JSON and Go](http://blog.golang.org/json-and-go)
- [Go Slices: usage and internals](http://blog.golang.org/go-slices-usage-and-internals)
- [Go Concurrency Patterns: Timing out, moving on](http://blog.golang.org/go-concurrency-patterns-timing-out-and)
- [Defer, Panic, and Recover](http://blog.golang.org/defer-panic-and-recover)
- [Share Memory By Communicating](http://blog.golang.org/share-memory-by-communicating)
- [JSON-RPC: a tale of interfaces](http://blog.golang.org/json-rpc-tale-of-interfaces)
Except as
[noted](https://developers.google.com/site-policies#restrictions), the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license](http://golang.org/LICENSE).
[Terms of Service](http://golang.org/doc/tos.html) \|
[Privacy Policy](http://www.google.com/intl/en/policies/privacy/) \|
[View the source code](https://go.googlesource.com/blog/) output.emphasis_underscores.golden 0000664 0000000 0000000 00000021373 14624642056 0041234 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language](http://golang.org/)
[Go](http://golang.org/)
▽
[Documents](http://golang.org/doc/) [Packages](http://golang.org/pkg/) [The Project](http://golang.org/project/) [Help](http://golang.org/help/) [Blog](http://blog.golang.org/)submit search
#### Next article
[Introducing Gofix](http://blog.golang.org/introducing-gofix)
#### Previous article
[Gobs of data](http://blog.golang.org/gobs-of-data)
#### Links
- [golang.org](http://golang.org/)
- [Install Go](http://golang.org/doc/install.html)
- [A Tour of Go](http://tour.golang.org/)
- [Go Documentation](http://golang.org/doc/)
- [Go Mailing List](http://groups.google.com/group/golang-nuts)
- [Go on Google+](http://plus.google.com/101406623878176903605)
- [Go+ Community](http://plus.google.com/communities/114112804251407510571)
- [Go on Twitter](http://twitter.com/golang)
[Blog index](http://blog.golang.org/index)
# [The Go Blog](http://blog.golang.org/)
### [Godoc: documenting Go code](http://blog.golang.org/godoc-documenting-go-code)
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc](https://golang.org/cmd/godoc/) documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation](https://golang.org/pkg/strings/#HasPrefix) to its
[implementation](https://golang.org/src/pkg/strings/strings.go#L493) with one click.
Godoc is conceptually related to Python's
[Docstring](http://www.python.org/dev/peps/pep-0257/) and Java's
[Javadoc](http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html), but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`](https://golang.org/pkg/fmt/#Fprint) function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`](https://golang.org/pkg/sort/) package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package](https://golang.org/pkg/encoding/gob/)'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go), which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list](https://golang.org/pkg/).
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package](https://golang.org/pkg/bytes/#pkg-note-BUG):
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library](https://golang.org/search?q=Deprecated:).
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go) for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation](https://golang.org/cmd/godoc/) for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push](http://blog.golang.org/h2push)
- [Introducing HTTP Tracing](http://blog.golang.org/http-tracing)
- [Testable Examples in Go](http://blog.golang.org/examples)
- [Generating code](http://blog.golang.org/generate)
- [Introducing the Go Race Detector](http://blog.golang.org/race-detector)
- [Go maps in action](http://blog.golang.org/go-maps-in-action)
- [go fmt your code](http://blog.golang.org/go-fmt-your-code)
- [Organizing Go code](http://blog.golang.org/organizing-go-code)
- [Debugging Go programs with the GNU Debugger](http://blog.golang.org/debugging-go-programs-with-gnu-debugger)
- [The Go image/draw package](http://blog.golang.org/go-imagedraw-package)
- [The Go image package](http://blog.golang.org/go-image-package)
- [The Laws of Reflection](http://blog.golang.org/laws-of-reflection)
- [Error handling and Go](http://blog.golang.org/error-handling-and-go)
- ["First Class Functions in Go"](http://blog.golang.org/first-class-functions-in-go-and-new-go)
- [Profiling Go Programs](http://blog.golang.org/profiling-go-programs)
- [A GIF decoder: an exercise in Go interfaces](http://blog.golang.org/gif-decoder-exercise-in-go-interfaces)
- [Introducing Gofix](http://blog.golang.org/introducing-gofix)
- [Gobs of data](http://blog.golang.org/gobs-of-data)
- [C? Go? Cgo!](http://blog.golang.org/c-go-cgo)
- [JSON and Go](http://blog.golang.org/json-and-go)
- [Go Slices: usage and internals](http://blog.golang.org/go-slices-usage-and-internals)
- [Go Concurrency Patterns: Timing out, moving on](http://blog.golang.org/go-concurrency-patterns-timing-out-and)
- [Defer, Panic, and Recover](http://blog.golang.org/defer-panic-and-recover)
- [Share Memory By Communicating](http://blog.golang.org/share-memory-by-communicating)
- [JSON-RPC: a tale of interfaces](http://blog.golang.org/json-rpc-tale-of-interfaces)
Except as
[noted](https://developers.google.com/site-policies#restrictions), the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license](http://golang.org/LICENSE).
[Terms of Service](http://golang.org/doc/tos.html) \|
[Privacy Policy](http://www.google.com/intl/en/policies/privacy/) \|
[View the source code](https://go.googlesource.com/blog/) output.inlined.golden 0000664 0000000 0000000 00000021373 14624642056 0036431 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language](http://golang.org/)
[Go](http://golang.org/)
▽
[Documents](http://golang.org/doc/) [Packages](http://golang.org/pkg/) [The Project](http://golang.org/project/) [Help](http://golang.org/help/) [Blog](http://blog.golang.org/)submit search
#### Next article
[Introducing Gofix](http://blog.golang.org/introducing-gofix)
#### Previous article
[Gobs of data](http://blog.golang.org/gobs-of-data)
#### Links
- [golang.org](http://golang.org/)
- [Install Go](http://golang.org/doc/install.html)
- [A Tour of Go](http://tour.golang.org/)
- [Go Documentation](http://golang.org/doc/)
- [Go Mailing List](http://groups.google.com/group/golang-nuts)
- [Go on Google+](http://plus.google.com/101406623878176903605)
- [Go+ Community](http://plus.google.com/communities/114112804251407510571)
- [Go on Twitter](http://twitter.com/golang)
[Blog index](http://blog.golang.org/index)
# [The Go Blog](http://blog.golang.org/)
### [Godoc: documenting Go code](http://blog.golang.org/godoc-documenting-go-code)
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc](https://golang.org/cmd/godoc/) documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation](https://golang.org/pkg/strings/#HasPrefix) to its
[implementation](https://golang.org/src/pkg/strings/strings.go#L493) with one click.
Godoc is conceptually related to Python's
[Docstring](http://www.python.org/dev/peps/pep-0257/) and Java's
[Javadoc](http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html), but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`](https://golang.org/pkg/fmt/#Fprint) function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`](https://golang.org/pkg/sort/) package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package](https://golang.org/pkg/encoding/gob/)'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go), which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list](https://golang.org/pkg/).
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package](https://golang.org/pkg/bytes/#pkg-note-BUG):
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library](https://golang.org/search?q=Deprecated:).
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go](https://golang.org/src/pkg/encoding/gob/doc.go) for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation](https://golang.org/cmd/godoc/) for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push](http://blog.golang.org/h2push)
- [Introducing HTTP Tracing](http://blog.golang.org/http-tracing)
- [Testable Examples in Go](http://blog.golang.org/examples)
- [Generating code](http://blog.golang.org/generate)
- [Introducing the Go Race Detector](http://blog.golang.org/race-detector)
- [Go maps in action](http://blog.golang.org/go-maps-in-action)
- [go fmt your code](http://blog.golang.org/go-fmt-your-code)
- [Organizing Go code](http://blog.golang.org/organizing-go-code)
- [Debugging Go programs with the GNU Debugger](http://blog.golang.org/debugging-go-programs-with-gnu-debugger)
- [The Go image/draw package](http://blog.golang.org/go-imagedraw-package)
- [The Go image package](http://blog.golang.org/go-image-package)
- [The Laws of Reflection](http://blog.golang.org/laws-of-reflection)
- [Error handling and Go](http://blog.golang.org/error-handling-and-go)
- ["First Class Functions in Go"](http://blog.golang.org/first-class-functions-in-go-and-new-go)
- [Profiling Go Programs](http://blog.golang.org/profiling-go-programs)
- [A GIF decoder: an exercise in Go interfaces](http://blog.golang.org/gif-decoder-exercise-in-go-interfaces)
- [Introducing Gofix](http://blog.golang.org/introducing-gofix)
- [Gobs of data](http://blog.golang.org/gobs-of-data)
- [C? Go? Cgo!](http://blog.golang.org/c-go-cgo)
- [JSON and Go](http://blog.golang.org/json-and-go)
- [Go Slices: usage and internals](http://blog.golang.org/go-slices-usage-and-internals)
- [Go Concurrency Patterns: Timing out, moving on](http://blog.golang.org/go-concurrency-patterns-timing-out-and)
- [Defer, Panic, and Recover](http://blog.golang.org/defer-panic-and-recover)
- [Share Memory By Communicating](http://blog.golang.org/share-memory-by-communicating)
- [JSON-RPC: a tale of interfaces](http://blog.golang.org/json-rpc-tale-of-interfaces)
Except as
[noted](https://developers.google.com/site-policies#restrictions), the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license](http://golang.org/LICENSE).
[Terms of Service](http://golang.org/doc/tos.html) \|
[Privacy Policy](http://www.google.com/intl/en/policies/privacy/) \|
[View the source code](https://go.googlesource.com/blog/) output.referenced_collapsed.golden 0000664 0000000 0000000 00000024137 14624642056 0041140 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language][]
[Go][]
▽
[Documents][] [Packages][] [The Project][] [Help][] [Blog][]submit search
#### Next article
[Introducing Gofix][]
#### Previous article
[Gobs of data][]
#### Links
- [golang.org][]
- [Install Go][]
- [A Tour of Go][]
- [Go Documentation][]
- [Go Mailing List][]
- [Go on Google+][]
- [Go+ Community][]
- [Go on Twitter][]
[Blog index][]
# [The Go Blog][]
### [Godoc: documenting Go code][]
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc][] documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation][] to its
[implementation][] with one click.
Godoc is conceptually related to Python's
[Docstring][] and Java's
[Javadoc][], but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`][] function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`][] package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package][]'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go][], which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list][].
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package][]:
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library][].
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go][] for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation][] for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push][]
- [Introducing HTTP Tracing][]
- [Testable Examples in Go][]
- [Generating code][]
- [Introducing the Go Race Detector][]
- [Go maps in action][]
- [go fmt your code][]
- [Organizing Go code][]
- [Debugging Go programs with the GNU Debugger][]
- [The Go image/draw package][]
- [The Go image package][]
- [The Laws of Reflection][]
- [Error handling and Go][]
- ["First Class Functions in Go"][]
- [Profiling Go Programs][]
- [A GIF decoder: an exercise in Go interfaces][]
- [Introducing Gofix][]
- [Gobs of data][]
- [C? Go? Cgo!][]
- [JSON and Go][]
- [Go Slices: usage and internals][]
- [Go Concurrency Patterns: Timing out, moving on][]
- [Defer, Panic, and Recover][]
- [Share Memory By Communicating][]
- [JSON-RPC: a tale of interfaces][]
Except as
[noted][], the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license][].
[Terms of Service][] \|
[Privacy Policy][] \|
[View the source code][]
[The Go Programming Language]: http://golang.org/
[Go]: http://golang.org/
[Documents]: http://golang.org/doc/
[Packages]: http://golang.org/pkg/
[The Project]: http://golang.org/project/
[Help]: http://golang.org/help/
[Blog]: http://blog.golang.org/
[Introducing Gofix]: http://blog.golang.org/introducing-gofix
[Gobs of data]: http://blog.golang.org/gobs-of-data
[golang.org]: http://golang.org/
[Install Go]: http://golang.org/doc/install.html
[A Tour of Go]: http://tour.golang.org/
[Go Documentation]: http://golang.org/doc/
[Go Mailing List]: http://groups.google.com/group/golang-nuts
[Go on Google+]: http://plus.google.com/101406623878176903605
[Go+ Community]: http://plus.google.com/communities/114112804251407510571
[Go on Twitter]: http://twitter.com/golang
[Blog index]: http://blog.golang.org/index
[The Go Blog]: http://blog.golang.org/
[Godoc: documenting Go code]: http://blog.golang.org/godoc-documenting-go-code
[godoc]: https://golang.org/cmd/godoc/
[documentation]: https://golang.org/pkg/strings/#HasPrefix
[implementation]: https://golang.org/src/pkg/strings/strings.go#L493
[Docstring]: http://www.python.org/dev/peps/pep-0257/
[Javadoc]: http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html
[`Fprint`]: https://golang.org/pkg/fmt/#Fprint
[`sort`]: https://golang.org/pkg/sort/
[gob package]: https://golang.org/pkg/encoding/gob/
[doc.go]: https://golang.org/src/pkg/encoding/gob/doc.go
[package list]: https://golang.org/pkg/
[bytes package]: https://golang.org/pkg/bytes/#pkg-note-BUG
[in the standard library]: https://golang.org/search?q=Deprecated:
[doc.go]: https://golang.org/src/pkg/encoding/gob/doc.go
[godoc documentation]: https://golang.org/cmd/godoc/
[HTTP/2 Server Push]: http://blog.golang.org/h2push
[Introducing HTTP Tracing]: http://blog.golang.org/http-tracing
[Testable Examples in Go]: http://blog.golang.org/examples
[Generating code]: http://blog.golang.org/generate
[Introducing the Go Race Detector]: http://blog.golang.org/race-detector
[Go maps in action]: http://blog.golang.org/go-maps-in-action
[go fmt your code]: http://blog.golang.org/go-fmt-your-code
[Organizing Go code]: http://blog.golang.org/organizing-go-code
[Debugging Go programs with the GNU Debugger]: http://blog.golang.org/debugging-go-programs-with-gnu-debugger
[The Go image/draw package]: http://blog.golang.org/go-imagedraw-package
[The Go image package]: http://blog.golang.org/go-image-package
[The Laws of Reflection]: http://blog.golang.org/laws-of-reflection
[Error handling and Go]: http://blog.golang.org/error-handling-and-go
["First Class Functions in Go"]: http://blog.golang.org/first-class-functions-in-go-and-new-go
[Profiling Go Programs]: http://blog.golang.org/profiling-go-programs
[A GIF decoder: an exercise in Go interfaces]: http://blog.golang.org/gif-decoder-exercise-in-go-interfaces
[Introducing Gofix]: http://blog.golang.org/introducing-gofix
[Gobs of data]: http://blog.golang.org/gobs-of-data
[C? Go? Cgo!]: http://blog.golang.org/c-go-cgo
[JSON and Go]: http://blog.golang.org/json-and-go
[Go Slices: usage and internals]: http://blog.golang.org/go-slices-usage-and-internals
[Go Concurrency Patterns: Timing out, moving on]: http://blog.golang.org/go-concurrency-patterns-timing-out-and
[Defer, Panic, and Recover]: http://blog.golang.org/defer-panic-and-recover
[Share Memory By Communicating]: http://blog.golang.org/share-memory-by-communicating
[JSON-RPC: a tale of interfaces]: http://blog.golang.org/json-rpc-tale-of-interfaces
[noted]: https://developers.google.com/site-policies#restrictions
[BSD license]: http://golang.org/LICENSE
[Terms of Service]: http://golang.org/doc/tos.html
[Privacy Policy]: http://www.google.com/intl/en/policies/privacy/
[View the source code]: https://go.googlesource.com/blog/ output.referenced_full.golden 0000664 0000000 0000000 00000022454 14624642056 0040134 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language][1]
[Go][2]
▽
[Documents][4] [Packages][5] [The Project][6] [Help][7] [Blog][8]submit search
#### Next article
[Introducing Gofix][9]
#### Previous article
[Gobs of data][10]
#### Links
- [golang.org][11]
- [Install Go][12]
- [A Tour of Go][13]
- [Go Documentation][14]
- [Go Mailing List][15]
- [Go on Google+][16]
- [Go+ Community][17]
- [Go on Twitter][18]
[Blog index][19]
# [The Go Blog][20]
### [Godoc: documenting Go code][21]
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc][22] documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation][23] to its
[implementation][24] with one click.
Godoc is conceptually related to Python's
[Docstring][25] and Java's
[Javadoc][26], but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`][27] function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`][28] package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package][29]'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go][30], which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list][31].
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package][32]:
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library][33].
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go][34] for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation][35] for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push][36]
- [Introducing HTTP Tracing][37]
- [Testable Examples in Go][38]
- [Generating code][39]
- [Introducing the Go Race Detector][40]
- [Go maps in action][41]
- [go fmt your code][42]
- [Organizing Go code][43]
- [Debugging Go programs with the GNU Debugger][44]
- [The Go image/draw package][45]
- [The Go image package][46]
- [The Laws of Reflection][47]
- [Error handling and Go][48]
- ["First Class Functions in Go"][49]
- [Profiling Go Programs][50]
- [A GIF decoder: an exercise in Go interfaces][51]
- [Introducing Gofix][52]
- [Gobs of data][53]
- [C? Go? Cgo!][54]
- [JSON and Go][55]
- [Go Slices: usage and internals][56]
- [Go Concurrency Patterns: Timing out, moving on][57]
- [Defer, Panic, and Recover][58]
- [Share Memory By Communicating][59]
- [JSON-RPC: a tale of interfaces][60]
Except as
[noted][61], the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license][62].
[Terms of Service][63] \|
[Privacy Policy][64] \|
[View the source code][65]
[1]: http://golang.org/
[2]: http://golang.org/
[4]: http://golang.org/doc/
[5]: http://golang.org/pkg/
[6]: http://golang.org/project/
[7]: http://golang.org/help/
[8]: http://blog.golang.org/
[9]: http://blog.golang.org/introducing-gofix
[10]: http://blog.golang.org/gobs-of-data
[11]: http://golang.org/
[12]: http://golang.org/doc/install.html
[13]: http://tour.golang.org/
[14]: http://golang.org/doc/
[15]: http://groups.google.com/group/golang-nuts
[16]: http://plus.google.com/101406623878176903605
[17]: http://plus.google.com/communities/114112804251407510571
[18]: http://twitter.com/golang
[19]: http://blog.golang.org/index
[20]: http://blog.golang.org/
[21]: http://blog.golang.org/godoc-documenting-go-code
[22]: https://golang.org/cmd/godoc/
[23]: https://golang.org/pkg/strings/#HasPrefix
[24]: https://golang.org/src/pkg/strings/strings.go#L493
[25]: http://www.python.org/dev/peps/pep-0257/
[26]: http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html
[27]: https://golang.org/pkg/fmt/#Fprint
[28]: https://golang.org/pkg/sort/
[29]: https://golang.org/pkg/encoding/gob/
[30]: https://golang.org/src/pkg/encoding/gob/doc.go
[31]: https://golang.org/pkg/
[32]: https://golang.org/pkg/bytes/#pkg-note-BUG
[33]: https://golang.org/search?q=Deprecated:
[34]: https://golang.org/src/pkg/encoding/gob/doc.go
[35]: https://golang.org/cmd/godoc/
[36]: http://blog.golang.org/h2push
[37]: http://blog.golang.org/http-tracing
[38]: http://blog.golang.org/examples
[39]: http://blog.golang.org/generate
[40]: http://blog.golang.org/race-detector
[41]: http://blog.golang.org/go-maps-in-action
[42]: http://blog.golang.org/go-fmt-your-code
[43]: http://blog.golang.org/organizing-go-code
[44]: http://blog.golang.org/debugging-go-programs-with-gnu-debugger
[45]: http://blog.golang.org/go-imagedraw-package
[46]: http://blog.golang.org/go-image-package
[47]: http://blog.golang.org/laws-of-reflection
[48]: http://blog.golang.org/error-handling-and-go
[49]: http://blog.golang.org/first-class-functions-in-go-and-new-go
[50]: http://blog.golang.org/profiling-go-programs
[51]: http://blog.golang.org/gif-decoder-exercise-in-go-interfaces
[52]: http://blog.golang.org/introducing-gofix
[53]: http://blog.golang.org/gobs-of-data
[54]: http://blog.golang.org/c-go-cgo
[55]: http://blog.golang.org/json-and-go
[56]: http://blog.golang.org/go-slices-usage-and-internals
[57]: http://blog.golang.org/go-concurrency-patterns-timing-out-and
[58]: http://blog.golang.org/defer-panic-and-recover
[59]: http://blog.golang.org/share-memory-by-communicating
[60]: http://blog.golang.org/json-rpc-tale-of-interfaces
[61]: https://developers.google.com/site-policies#restrictions
[62]: http://golang.org/LICENSE
[63]: http://golang.org/doc/tos.html
[64]: http://www.google.com/intl/en/policies/privacy/
[65]: https://go.googlesource.com/blog/ output.referenced_shortcut.golden 0000664 0000000 0000000 00000023737 14624642056 0041052 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/blog.golang.org Godoc: documenting Go code - The Go Blog
[The Go Programming Language]
[Go]
▽
[Documents] [Packages] [The Project] [Help] [Blog]submit search
#### Next article
[Introducing Gofix]
#### Previous article
[Gobs of data]
#### Links
- [golang.org]
- [Install Go]
- [A Tour of Go]
- [Go Documentation]
- [Go Mailing List]
- [Go on Google+]
- [Go+ Community]
- [Go on Twitter]
[Blog index]
# [The Go Blog]
### [Godoc: documenting Go code]
31 March 2011
The Go project takes documentation seriously. Documentation is a huge part of making software accessible and maintainable.
Of course it must be well-written and accurate, but it also must be easy to write and to maintain. Ideally, it
should be coupled to the code itself so the documentation evolves along with the code. The easier it is for programmers
to produce good documentation, the better for everyone.
To that end, we have developed the
[godoc] documentation tool. This article describes godoc's approach to documentation, and explains how
you can use our conventions and tools to write good documentation for your own projects.
Godoc parses Go source code - including comments - and produces documentation as HTML or plain text. The end result is documentation
tightly coupled with the code it documents. For example, through godoc's web interface you can navigate from
a function's
[documentation] to its
[implementation] with one click.
Godoc is conceptually related to Python's
[Docstring] and Java's
[Javadoc], but its design is simpler. The comments read by godoc are not language constructs (as with Docstring)
nor must they have their own machine-readable syntax (as with Javadoc). Godoc comments are just good comments,
the sort you would want to read even if godoc didn't exist.
The convention is simple: to document a type, variable, constant, function, or even a package, write a regular comment directly
preceding its declaration, with no intervening blank line. Godoc will then present that comment as text alongside
the item it documents. For example, this is the documentation for the
`fmt` package's
[`Fprint`] function:
```
// Fprint formats using the default formats for its operands and writes to w.
// Spaces are added between operands when neither is a string.
// It returns the number of bytes written and any write error encountered.
func Fprint(w io.Writer, a ...interface{}) (n int, err error) {
```
Notice this comment is a complete sentence that begins with the name of the element it describes. This important convention
allows us to generate documentation in a variety of formats, from plain text to HTML to UNIX man pages, and makes
it read better when tools truncate it for brevity, such as when they extract the first line or sentence.
Comments on package declarations should provide general package documentation. These comments can be short, like the
[`sort`] package's brief description:
```
// Package sort provides primitives for sorting slices and user-defined
// collections.
package sort
```
They can also be detailed like the
[gob package]'s overview. That package uses another convention for packages that need large amounts of
introductory documentation: the package comment is placed in its own file,
[doc.go], which contains only those comments and a package clause.
When writing package comments of any size, keep in mind that their first sentence will appear in godoc's
[package list].
Comments that are not adjacent to a top-level declaration are omitted from godoc's output, with one notable exception.
Top-level comments that begin with the word
`"BUG(who)”` are recognized as known bugs, and included in the "Bugs” section of the package documentation. The "who”
part should be the user name of someone who could provide more information. For example, this is a known issue
from the
[bytes package]:
```
// BUG(r): The rule Title uses for word boundaries does not handle Unicode punctuation properly.
```
Sometimes a struct field, function, type, or even a whole package becomes redundant or unnecessary, but must be kept for
compatibility with existing programs. To signal that an identifier should not be used, add a paragraph to its
doc comment that begins with "Deprecated:" followed by some information about the deprecation. There
are a few examples
[in the standard library].
There are a few formatting rules that Godoc uses when converting comments to HTML:
- Subsequent lines of text are considered part of the same paragraph; you must leave a blank line to separate paragraphs.
- Pre-formatted text must be indented relative to the surrounding comment text (see gob's
[doc.go] for an example).
- URLs will be converted to HTML links; no special markup is necessary.
Note that none of these rules requires you to do anything out of the ordinary.
In fact, the best thing about godoc's minimal approach is how easy it is to use. As a result, a lot of Go code, including
all of the standard library, already follows the conventions.
Your own code can present good documentation just by having comments as described above. Any Go packages installed inside
`$GOROOT/src/pkg` and any
`GOPATH` work spaces will already be accessible via godoc's command-line and HTTP interfaces, and you can specify
additional paths for indexing via the
`-path` flag or just by running
`"godoc ."` in the source directory. See the
[godoc documentation] for more details.
By Andrew Gerrand
## Related articles
- [HTTP/2 Server Push]
- [Introducing HTTP Tracing]
- [Testable Examples in Go]
- [Generating code]
- [Introducing the Go Race Detector]
- [Go maps in action]
- [go fmt your code]
- [Organizing Go code]
- [Debugging Go programs with the GNU Debugger]
- [The Go image/draw package]
- [The Go image package]
- [The Laws of Reflection]
- [Error handling and Go]
- ["First Class Functions in Go"]
- [Profiling Go Programs]
- [A GIF decoder: an exercise in Go interfaces]
- [Introducing Gofix]
- [Gobs of data]
- [C? Go? Cgo!]
- [JSON and Go]
- [Go Slices: usage and internals]
- [Go Concurrency Patterns: Timing out, moving on]
- [Defer, Panic, and Recover]
- [Share Memory By Communicating]
- [JSON-RPC: a tale of interfaces]
Except as
[noted], the content of this page is licensed under the Creative Commons Attribution 3.0 License,
and code is licensed under a
[BSD license].
[Terms of Service] \|
[Privacy Policy] \|
[View the source code]
[The Go Programming Language]: http://golang.org/
[Go]: http://golang.org/
[Documents]: http://golang.org/doc/
[Packages]: http://golang.org/pkg/
[The Project]: http://golang.org/project/
[Help]: http://golang.org/help/
[Blog]: http://blog.golang.org/
[Introducing Gofix]: http://blog.golang.org/introducing-gofix
[Gobs of data]: http://blog.golang.org/gobs-of-data
[golang.org]: http://golang.org/
[Install Go]: http://golang.org/doc/install.html
[A Tour of Go]: http://tour.golang.org/
[Go Documentation]: http://golang.org/doc/
[Go Mailing List]: http://groups.google.com/group/golang-nuts
[Go on Google+]: http://plus.google.com/101406623878176903605
[Go+ Community]: http://plus.google.com/communities/114112804251407510571
[Go on Twitter]: http://twitter.com/golang
[Blog index]: http://blog.golang.org/index
[The Go Blog]: http://blog.golang.org/
[Godoc: documenting Go code]: http://blog.golang.org/godoc-documenting-go-code
[godoc]: https://golang.org/cmd/godoc/
[documentation]: https://golang.org/pkg/strings/#HasPrefix
[implementation]: https://golang.org/src/pkg/strings/strings.go#L493
[Docstring]: http://www.python.org/dev/peps/pep-0257/
[Javadoc]: http://www.oracle.com/technetwork/java/javase/documentation/index-jsp-135444.html
[`Fprint`]: https://golang.org/pkg/fmt/#Fprint
[`sort`]: https://golang.org/pkg/sort/
[gob package]: https://golang.org/pkg/encoding/gob/
[doc.go]: https://golang.org/src/pkg/encoding/gob/doc.go
[package list]: https://golang.org/pkg/
[bytes package]: https://golang.org/pkg/bytes/#pkg-note-BUG
[in the standard library]: https://golang.org/search?q=Deprecated:
[doc.go]: https://golang.org/src/pkg/encoding/gob/doc.go
[godoc documentation]: https://golang.org/cmd/godoc/
[HTTP/2 Server Push]: http://blog.golang.org/h2push
[Introducing HTTP Tracing]: http://blog.golang.org/http-tracing
[Testable Examples in Go]: http://blog.golang.org/examples
[Generating code]: http://blog.golang.org/generate
[Introducing the Go Race Detector]: http://blog.golang.org/race-detector
[Go maps in action]: http://blog.golang.org/go-maps-in-action
[go fmt your code]: http://blog.golang.org/go-fmt-your-code
[Organizing Go code]: http://blog.golang.org/organizing-go-code
[Debugging Go programs with the GNU Debugger]: http://blog.golang.org/debugging-go-programs-with-gnu-debugger
[The Go image/draw package]: http://blog.golang.org/go-imagedraw-package
[The Go image package]: http://blog.golang.org/go-image-package
[The Laws of Reflection]: http://blog.golang.org/laws-of-reflection
[Error handling and Go]: http://blog.golang.org/error-handling-and-go
["First Class Functions in Go"]: http://blog.golang.org/first-class-functions-in-go-and-new-go
[Profiling Go Programs]: http://blog.golang.org/profiling-go-programs
[A GIF decoder: an exercise in Go interfaces]: http://blog.golang.org/gif-decoder-exercise-in-go-interfaces
[Introducing Gofix]: http://blog.golang.org/introducing-gofix
[Gobs of data]: http://blog.golang.org/gobs-of-data
[C? Go? Cgo!]: http://blog.golang.org/c-go-cgo
[JSON and Go]: http://blog.golang.org/json-and-go
[Go Slices: usage and internals]: http://blog.golang.org/go-slices-usage-and-internals
[Go Concurrency Patterns: Timing out, moving on]: http://blog.golang.org/go-concurrency-patterns-timing-out-and
[Defer, Panic, and Recover]: http://blog.golang.org/defer-panic-and-recover
[Share Memory By Communicating]: http://blog.golang.org/share-memory-by-communicating
[JSON-RPC: a tale of interfaces]: http://blog.golang.org/json-rpc-tale-of-interfaces
[noted]: https://developers.google.com/site-policies#restrictions
[BSD license]: http://golang.org/LICENSE
[Terms of Service]: http://golang.org/doc/tos.html
[Privacy Policy]: http://www.google.com/intl/en/policies/privacy/
[View the source code]: https://go.googlesource.com/blog/ golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/golang.org/ 0000775 0000000 0000000 00000000000 14624642056 0031405 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000003130 14624642056 0034315 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/golang.org
output.default.golden 0000664 0000000 0000000 00000002433 14624642056 0035505 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/golang.org The Go Programming Language
...
[The Go Programming Language](http://golang.org/)
[Go](http://golang.org/)
▽
[Documents](http://golang.org/doc/) [Packages](http://golang.org/pkg/) [The Project](http://golang.org/project/) [Help](http://golang.org/help/) [Blog](http://golang.org/blog/) [Play](http://play.golang.org/ "Show Go Playground") submit search
RunFormatShare
Pop-out
Try Go
```
Hello, 世界
```
RunShare [Tour](http://tour.golang.org/ "Learn Go from your browser")
Hello, World!Conway's Game of LifeFibonacci ClosurePeano IntegersConcurrent piConcurrent Prime SievePeg Solitaire SolverTree Comparison
Go is an open source programming language that makes it easy to build
simple, reliable, and efficient software.
[Download Go\
Binary distributions available for\
\
Linux, Mac OS X, Windows, and more.](http://golang.org/dl/)
Featured video
Featured articles
[Read more](http://blog.golang.org/)
Build version go1.10.2.
Except as [noted](https://developers.google.com/site-policies#restrictions),
the content of this page is licensed under the
Creative Commons Attribution 3.0 License,
and code is licensed under a [BSD license](http://golang.org/LICENSE).
[Terms of Service](http://golang.org/doc/tos.html) \|
[Privacy Policy](http://www.google.com/intl/en/policies/privacy/) golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/ 0000775 0000000 0000000 00000000000 14624642056 0031215 5 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/github_about/ 0000775 0000000 0000000 00000000000 14624642056 0033671 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000001300 14624642056 0036576 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/github_about
About
⚙️
Convert HTML to Markdown. Even works with whole websites.
The MJML tool provides a CLI you can use to transform MJML into HTML:
bash
$ mjml input.mjml -o output.html
Curiously, all text elements (paragraphs and headings) use the same tag,
<mj-text>. You can
create headings by applying cosmetic styles as inline attributes, like:
html
<mj-text
align="center"
font-size="32px"
font-weight="bold"
color="#FF0000"
>
func HasPrefix(s, prefix []byte) bool
HTML
<p>Using CSS to change the font color is easy.</p>
<pre>
body {
color: red;
}
</pre>
Copy to Clipboard
Using CSS to change the font color is easy.
body {
color: red;
}
复制代码const Pi = 3.14159
const Pi = 3.14159
复制代码var n intf(n + 5) // 无类型的数字型常量 “5” 它的类型在这里变成了 int
mkdir hugo-contrib
cd hugo-contrib
git clone git@github.com:gohugoio/hugo.git
Save the changes and run your first bob command:
bob build
To confirm that everything went fine, just run:
bob --version
Copy
For instance, MetaMask displays your recent transactions by making an API call
to etherscan:
GET https://api.etherscan.io/api?module=account&address=0x0208376c899fdaEbA530570c008C4323803AA9E8&offset=40&order=desc&action=txlist&tag=latest&page=1 HTTP/2.0
…displays your account balance by making an API call to Infura:
After it was working, I started adding as many different programming languages
as I could. As you can see from this excerpt of my project’s version history,
I got a little overexcited:
languages
---------
2020-06-05 df9ba38 Initial commit 0
2020-06-05 5e3a4a4 Install some packages into a Docker image 0
2020-06-05 e937c8f Simple Express server with "Hello world" 0
2020-06-06 0961498 Embed terminal on frontend app 0
2020-06-06 c66cf63 Embed Monaco editor on frontend 0
2020-06-06 27ab1f7 Add "run" button 0
2020-06-06 f417858 You can run Python code now 1
2020-06-07 d543081 You can run many languages now 8
2020-06-07 e2a3e71 All languages 17 working now 17
2020-06-07 473c50c ALL THE LANGUAGES 25
2020-06-08 3718315 even more languages 33
2020-06-08 548c1c1 repl.it superiority!! 38
2020-06-08 1ae424f More languages, we need all the languages 48
2020-06-09 c34ccf2 A lot more languages 77
2020-06-09 846caf2 At this point the number of languages is absurd 79
Disassembling the now-less-obfuscated dump reveals that one of the addresses
has a label pulled out of somewhere! It’s strlen? Going down the
call stack the next one is labeled vscan_fn and after that the
labels end, tho I’m fairly confident it’s
sscanf.
<p>This is the <code>Panel</code> constructor:</p>
<pre><code>function Panel(element, canClose, closeHandler) {
this.element = element;
this.canClose = canClose;
this.closeHandler = function () { if (closeHandler) closeHandler() };
}</code></pre>
Run code snippet
Hide results
Expand snippet
Problem with <pre> is it modifies whitespace processing
as well: all spaces are preserved, and wrapping is switched off. Unless
there's a way to switch this off?
Use the <textarea> element to share code,
like so:
<textarea class="code" contenteditable="true" spellcheck="false" aria-label='Code Sample'>
My Sample Bookmark:
<a href="#bookmark1" id="b1" title="View my bookmark" target="_blank" rel="noreferrer nofollow noopener" accesskey="a" tabindex="0" aria-label="Bookmark">Got to My Bookmark</a>
</textarea>
linux-nfs.vger.kernel.org archive mirror
help / color / mirror / Atom feed
From: Leon Romanovsky <leon@kernel.org>
To: "J. Bruce Fields" <bfields@fieldses.org>
Cc: Greg KH <gregkh@linuxfoundation.org>,
Aditya Pakki <pakki001@umn.edu>,
Chuck Lever <chuck.lever@oracle.com>,
Trond Myklebust <trond.myklebust@hammerspace.com>,
Anna Schumaker <anna.schumaker@netapp.com>,
"David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>,
Dave Wysochanski <dwysocha@redhat.com>,
linux-nfs@vger.kernel.org, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: Re: [PATCH] SUNRPC: Add a check for gss_release_msg
Date: Wed, 21 Apr 2021 08:10:25 +0300 [thread overview]
Message-ID: <YH+zwQgBBGUJdiVK@unreal> (raw)
In-Reply-To: <20210420171008.GB4017@fieldses.org>
On Tue, Apr 20, 2021 at 01:10:08PM -0400, J. Bruce Fields wrote:
> On Tue, Apr 20, 2021 at 09:15:23AM +0200, Greg KH wrote:
> > If you look at the code, this is impossible to have happen.
> >
> > Please stop submitting known-invalid patches. Your professor is playing
> > around with the review process in order to achieve a paper in some
> > strange and bizarre way.
> >
> > This is not ok, it is wasting our time, and we will have to report this,
> > AGAIN, to your university...
>
> What's the story here?
Those commits are part of the following research:
https://github.com/QiushiWu/QiushiWu.github.io/blob/main/papers/OpenSourceInsecurity.pdf
They introduce kernel bugs on purpose. Yesterday, I took a look on 4
accepted patches from Aditya and 3 of them added various severity security
"holes".
Thanks
>
> --b.
next prev parent reply other threads:[~2021-04-21 5:10 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-07 0:16 Aditya Pakki
2021-04-07 15:34 ` J. Bruce Fields
2021-04-08 15:01 ` Trond Myklebust
2021-04-08 15:24 ` Olga Kornievskaia
2021-04-08 16:02 ` Trond Myklebust
2021-04-20 7:15 ` Greg KH
2021-04-20 17:10 ` J. Bruce Fields
2021-04-21 5:10 ` Leon Romanovsky [this message]
2021-04-21 5:43 ` Greg KH
2021-04-21 6:08 ` Leon Romanovsky
[not found] ` <CA+EnHHSw4X+ubOUNYP2zXNpu70G74NN1Sct2Zin6pRgq--TqhA@mail.gmail.com>
2021-04-21 8:15 ` Greg KH
2021-04-21 10:07 ` Sudip Mukherjee
2021-04-21 10:21 ` Greg KH
2021-04-21 11:58 ` Shelat, Abhi
2021-04-21 12:08 ` Greg KH
2021-04-21 12:19 ` Leon Romanovsky
2021-04-21 13:11 ` Trond Myklebust
2021-04-21 13:20 ` Leon Romanovsky
2021-04-21 13:42 ` Steven Rostedt
2021-04-21 13:21 ` gregkh
2021-04-21 13:34 ` Leon Romanovsky
2021-04-21 13:50 ` gregkh
2021-04-21 14:12 ` Leon Romanovsky
2021-04-21 18:50 ` Alexander Grund
2021-04-21 13:37 ` J. Bruce Fields
2021-04-21 13:49 ` Leon Romanovsky
2021-04-21 13:56 ` J. Bruce Fields
2021-04-22 19:39 ` J. Bruce Fields
2021-04-23 17:25 ` Leon Romanovsky
2021-04-23 18:07 ` J. Bruce Fields
2021-04-23 19:29 ` Leon Romanovsky
2021-04-23 21:48 ` J. Bruce Fields
2021-04-24 7:21 ` Leon Romanovsky
2021-04-24 18:34 ` Al Viro
2021-04-24 21:34 ` J. Bruce Fields
2021-04-25 0:41 ` Theodore Ts'o
2021-04-25 6:29 ` Greg KH
[not found] ` <20210426133605.GD21222@fieldses.org>
2021-04-26 13:47 ` J. Bruce Fields
2021-04-22 8:10 ` Sudip Mukherjee
2021-04-22 8:27 ` Greg KH
2021-04-21 12:51 ` Anna Schumaker
2021-04-21 14:15 ` Leon Romanovsky
2021-04-21 15:48 ` Theodore Ts'o
2021-04-21 17:34 ` Mike Rapoport
2021-04-22 3:57 ` Leon Romanovsky
2021-04-21 22:52 ` Guenter Roeck
[not found] <CAHr+ZK-ayy2vku9ovuSB4egtOxrPEKxCdVQN3nFqMK07+K5_8g@mail.gmail.com>
2021-04-21 19:49 ` Theodore Ts'o
2021-04-22 7:50 ` Eric Biggers
2021-04-21 20:27 Weikeng Chen
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=YH+zwQgBBGUJdiVK@unreal \
--to=leon@kernel.org \
--cc=anna.schumaker@netapp.com \
--cc=bfields@fieldses.org \
--cc=chuck.lever@oracle.com \
--cc=davem@davemloft.net \
--cc=dwysocha@redhat.com \
--cc=gregkh@linuxfoundation.org \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=netdev@vger.kernel.org \
--cc=pakki001@umn.edu \
--cc=trond.myklebust@hammerspace.com \
--subject='Re: [PATCH] SUNRPC: Add a check for gss_release_msg' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).
octosql "SELECT email, COUNT(*) as invoice_count
FROM invoices.csv JOIN mydb.customers ON invoices.customer_id = customers.id
WHERE first_name <= 'D'
GROUP BY email
ORDER BY invoice_count DESC" --explain 1
If you really want this functionality, you can implement it by using the
following function:
function sleepSync(timeout) {
const sab = new SharedArrayBuffer(1024);
const int32 = new Int32Array(sab);
Atomics.wait(int32, 0, 0, timeout);
}
It doesn’t get much simpler than this. You’ll notice we only define a main()
function followed by a println to stdout.
fn main() {
println!("Hello, world!");
}
# Ruby knows what you
# mean, even if you
# want to do math on
# an entire Array
cities = %w[ London
Oslo
Paris
Amsterdam
Berlin ]
visited = %w[Berlin Oslo]
puts "I still need " +
"to visit the " +
"following cities:",
cities - visited
The equivalent portion is just these two lines:
var btn = new Gtk.Button.with_label("Hello World");
btn.click.connect(win.close);
The MJML tool provides a CLI you can use to transform MJML into HTML:
bash
$ mjml input.mjml -o output.html
Curiously, all text elements (paragraphs and headings) use the same tag,
<mj-text>. You can
create headings by applying cosmetic styles as inline attributes, like:
<p>Using CSS to change the font color is easy.</p><pre>
body {
color: red;
}
</pre>
Using CSS to change the font color is easy.
body {
color: red;
}
constPi=3.14159
constPi=3.14159
var n int
f(n +5)// 无类型的数字型常量 “5” 它的类型在这里变成了 int
mkdir hugo-contrib cd hugo-contrib git clone git@github.com:gohugoio/hugo.git
Save the changes and run your first bob command:
bob build
To confirm that everything went fine, just run:
bob --version
For instance, MetaMask displays your recent transactions by making an API call
to etherscan:
GET https://api.etherscan.io/api?module=account&address=0x0208376c899fdaEbA530570c008C4323803AA9E8&offset=40&order=desc&action=txlist&tag=latest&page=1 HTTP/2.0
…displays your account balance by making an API call to Infura:
After it was working, I started adding as many different programming languages
as I could. As you can see from this excerpt of my project’s version history,
I got a little overexcited:
languages
---------
2020-06-05 df9ba38 Initial commit 0
2020-06-05 5e3a4a4 Install some packages into a Docker image 0
2020-06-05 e937c8f Simple Express server with "Hello world" 0
2020-06-06 0961498 Embed terminal on frontend app 0
2020-06-06 c66cf63 Embed Monaco editor on frontend 0
2020-06-06 27ab1f7 Add "run" button 0
2020-06-06 f417858 You can run Python code now 1
2020-06-07 d543081 You can run many languages now 8
2020-06-07 e2a3e71 All languages 17 working now 17
2020-06-07 473c50c ALL THE LANGUAGES 25
2020-06-08 3718315 even more languages 33
2020-06-08 548c1c1 repl.it superiority!! 38
2020-06-08 1ae424f More languages, we need all the languages 48
2020-06-09 c34ccf2 A lot more languages 77
2020-06-09 846caf2 At this point the number of languages is absurd 79
Disassembling the now-less-obfuscated dump reveals that one of the addresses
has a label pulled out of somewhere! It’s strlen? Going down the
call stack the next one is labeled vscan_fn and after that the
labels end, tho I’m fairly confident it’s
sscanf.
Problem with <pre> is it modifies whitespace processing
as well: all spaces are preserved, and wrapping is switched off. Unless
there's a way to switch this off?
Use the <textarea> element to share code,
like so:
<textareaclass="code"contenteditable="true"spellcheck="false"aria-label='Code Sample'>
My Sample Bookmark:
<ahref="#bookmark1"id="b1"title="View my bookmark"target="_blank"rel="noreferrer nofollow noopener"accesskey="a"tabindex="0"aria-label="Bookmark">Got to My Bookmark</a></textarea>
From: Leon Romanovsky <leon@kernel.org>
To: "J. Bruce Fields" <bfields@fieldses.org>
Cc: Greg KH <gregkh@linuxfoundation.org>,
Aditya Pakki <pakki001@umn.edu>,
Chuck Lever <chuck.lever@oracle.com>,
Trond Myklebust <trond.myklebust@hammerspace.com>,
Anna Schumaker <anna.schumaker@netapp.com>,
"David S. Miller" <davem@davemloft.net>,
Jakub Kicinski <kuba@kernel.org>,
Dave Wysochanski <dwysocha@redhat.com>,
linux-nfs@vger.kernel.org, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: Re: [PATCH] SUNRPC: Add a check for gss_release_msg
Date: Wed, 21 Apr 2021 08:10:25 +0300 [thread overview]
Message-ID: <YH+zwQgBBGUJdiVK@unreal> (raw)
In-Reply-To: <20210420171008.GB4017@fieldses.org>
On Tue, Apr 20, 2021 at 01:10:08PM -0400, J. Bruce Fields wrote:
> On Tue, Apr 20, 2021 at 09:15:23AM +0200, Greg KH wrote:
> > If you look at the code, this is impossible to have happen.
> >
> > Please stop submitting known-invalid patches. Your professor is playing
> > around with the review process in order to achieve a paper in some
> > strange and bizarre way.
> >
> > This is not ok, it is wasting our time, and we will have to report this,
> > AGAIN, to your university...
>
> What's the story here?
Those commits are part of the following research:
https://github.com/QiushiWu/QiushiWu.github.io/blob/main/papers/OpenSourceInsecurity.pdf
They introduce kernel bugs on purpose. Yesterday, I took a look on 4
accepted patches from Aditya and 3 of them added various severity security
"holes".
Thanks
>
> --b.
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=YH+zwQgBBGUJdiVK@unreal \
--to=leon@kernel.org \
--cc=anna.schumaker@netapp.com \
--cc=bfields@fieldses.org \
--cc=chuck.lever@oracle.com \
--cc=davem@davemloft.net \
--cc=dwysocha@redhat.com \
--cc=gregkh@linuxfoundation.org \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=netdev@vger.kernel.org \
--cc=pakki001@umn.edu \
--cc=trond.myklebust@hammerspace.com \
--subject='Re: [PATCH] SUNRPC: Add a check for gss_release_msg' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).
octosql "SELECT email, COUNT(*) as invoice_count FROM invoices.csv JOIN mydb.customers ON invoices.customer_id = customers.id WHERE first_name <= 'D' GROUP BY email ORDER BY invoice_count DESC" --explain 1
If you really want this functionality, you can implement it by using the
following function:
functionsleepSync(timeout){const sab =newSharedArrayBuffer(1024);const int32 =newInt32Array(sab);
Atomics.wait(int32,0,0, timeout);}
It doesn’t get much simpler than this. You’ll notice we only define a main()
function followed by a println to stdout.
fnmain(){println!("Hello, world!");}
# Ruby knows what you# mean, even if you# want to do math on# an entire Arraycities=%w[ London
Oslo
Paris
Amsterdam
Berlin ]visited=%w[Berlin Oslo]puts"I still need "+"to visit the "+"following cities:",cities-visited
The equivalent portion is just these two lines:
var btn = new Gtk.Button.with_label("Hello World");
btn.click.connect(win.close);
output.default.golden 0000664 0000000 0000000 00000030563 14624642056 0037102 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/pre_code The MJML tool provides a CLI you can use to transform MJML into HTML:
```
bash
$ mjml input.mjml -o output.html
```
Curiously, all text elements (paragraphs and headings) use the same tag,
``. You can
create headings by applying cosmetic styles as inline attributes, like:
```
html
```
* * *
```
func HasPrefix(s, prefix []byte) bool
```
* * *
#### HTML
```
Using CSS to change the font color is easy.
body {
color: red;
}
```
Copy to Clipboard
Using CSS to change the font color is easy.
```
body {
color: red;
}
```
* * *
```lang-go
复制代码const Pi = 3.14159
```
`const Pi = 3.14159`
```lang-go
复制代码var n intf(n + 5) // 无类型的数字型常量 “5” 它的类型在这里变成了 int
```
* * *
```
mkdir hugo-contrib
cd hugo-contrib
git clone git@github.com:gohugoio/hugo.git
```
Save the changes and run your first bob command:
```
bob build
```
* * *
To confirm that everything went fine, just run:
```codeBlockLines_39YC
bob --version
```
Copy
* * *
For instance, MetaMask displays your recent transactions by making an API call
to etherscan:
```
GET https://api.etherscan.io/api?module=account&address=0x0208376c899fdaEbA530570c008C4323803AA9E8&offset=40&order=desc&action=txlist&tag=latest&page=1 HTTP/2.0
```
…displays your account balance by making an API call to Infura:
```
POST https://mainnet.infura.io/v3/d039103314584a379e33c21fbe89b6cb HTTP/2.0
{
"id": 2628746552039525,
"jsonrpc": "2.0",
"method": "eth_getBalance",
"params": [
"0x0208376c899fdaEbA530570c008C4323803AA9E8",
"latest"
]
}
```
* * *
After it was working, I started adding as many different programming languages
as I could. As you can see from this excerpt of my project’s version history,
I got a little overexcited:
```
languages
---------
2020-06-05 df9ba38 Initial commit 0
2020-06-05 5e3a4a4 Install some packages into a Docker image 0
2020-06-05 e937c8f Simple Express server with "Hello world" 0
2020-06-06 0961498 Embed terminal on frontend app 0
2020-06-06 c66cf63 Embed Monaco editor on frontend 0
2020-06-06 27ab1f7 Add "run" button 0
2020-06-06 f417858 You can run Python code now 1
2020-06-07 d543081 You can run many languages now 8
2020-06-07 e2a3e71 All languages 17 working now 17
2020-06-07 473c50c ALL THE LANGUAGES 25
2020-06-08 3718315 even more languages 33
2020-06-08 548c1c1 repl.it superiority!! 38
2020-06-08 1ae424f More languages, we need all the languages 48
2020-06-09 c34ccf2 A lot more languages 77
2020-06-09 846caf2 At this point the number of languages is absurd 79
```
* * *
Disassembling the now-less-obfuscated dump reveals that one of the addresses
has a label pulled out of somewhere! It’s `strlen`? Going down the
call stack the next one is labeled `vscan_fn` and after that the
labels end, tho I’m fairly confident it’s
[`sscanf`](https://github.com/chakra-core/ChakraCore/blob/master/pal/src/safecrt/sscanf.c#L47).
```
1
2
3
4
5
6
7
8
9
10
11
```
```
...,
{
"key": "WP_WCT_TINT_21_t2_v9_n2",
"price": 45000,
"statName": "CHAR_KIT_FM_PURCHASE20",
"storageType": "BITFIELD",
"bitShift": 7,
"bitSize": 1,
"category": ["CATEGORY_WEAPON_MOD"]
},
...
```
* * *
```hljs xml
This is the Panel constructor:
function Panel(element, canClose, closeHandler) {
this.element = element;
this.canClose = canClose;
this.closeHandler = function () { if (closeHandler) closeHandler() };
}
```
Run code snippet
Hide results
Expand snippet
Problem with `
` is it modifies whitespace processing
as well: all spaces are preserved, and wrapping is switched off. Unless
there's a way to switch this off?
**Use the `
```
* * *
* * *
```
linux-nfs.vger.kernel.org archive mirror
help / color / mirror / Atom feed
```
```
From: Leon Romanovsky
To: "J. Bruce Fields"
Cc: Greg KH ,
Aditya Pakki ,
Chuck Lever ,
Trond Myklebust ,
Anna Schumaker ,
"David S. Miller" ,
Jakub Kicinski ,
Dave Wysochanski ,
linux-nfs@vger.kernel.org, netdev@vger.kernel.org,
linux-kernel@vger.kernel.org
Subject: Re: [PATCH] SUNRPC: Add a check for gss_release_msg
Date: Wed, 21 Apr 2021 08:10:25 +0300 [thread overview]
Message-ID: (raw)
In-Reply-To: <20210420171008.GB4017@fieldses.org>
On Tue, Apr 20, 2021 at 01:10:08PM -0400, J. Bruce Fields wrote:
> On Tue, Apr 20, 2021 at 09:15:23AM +0200, Greg KH wrote:
> > If you look at the code, this is impossible to have happen.
> >
> > Please stop submitting known-invalid patches. Your professor is playing
> > around with the review process in order to achieve a paper in some
> > strange and bizarre way.
> >
> > This is not ok, it is wasting our time, and we will have to report this,
> > AGAIN, to your university...
>
> What's the story here?
Those commits are part of the following research:
https://github.com/QiushiWu/QiushiWu.github.io/blob/main/papers/OpenSourceInsecurity.pdf
They introduce kernel bugs on purpose. Yesterday, I took a look on 4
accepted patches from Aditya and 3 of them added various severity security
"holes".
Thanks
>
> --b.
```
* * *
```
next prev parent reply other threads:[~2021-04-21 5:10 UTC|newest]
Thread overview: 49+ messages / expand[flat|nested] mbox.gz Atom feed top
2021-04-07 0:16 Aditya Pakki
2021-04-07 15:34 ` J. Bruce Fields
2021-04-08 15:01 ` Trond Myklebust
2021-04-08 15:24 ` Olga Kornievskaia
2021-04-08 16:02 ` Trond Myklebust
2021-04-20 7:15 ` Greg KH
2021-04-20 17:10 ` J. Bruce Fields
2021-04-21 5:10 ` Leon Romanovsky [this message]
2021-04-21 5:43 ` Greg KH
2021-04-21 6:08 ` Leon Romanovsky
[not found] `
2021-04-21 8:15 ` Greg KH
2021-04-21 10:07 ` Sudip Mukherjee
2021-04-21 10:21 ` Greg KH
2021-04-21 11:58 ` Shelat, Abhi
2021-04-21 12:08 ` Greg KH
2021-04-21 12:19 ` Leon Romanovsky
2021-04-21 13:11 ` Trond Myklebust
2021-04-21 13:20 ` Leon Romanovsky
2021-04-21 13:42 ` Steven Rostedt
2021-04-21 13:21 ` gregkh
2021-04-21 13:34 ` Leon Romanovsky
2021-04-21 13:50 ` gregkh
2021-04-21 14:12 ` Leon Romanovsky
2021-04-21 18:50 ` Alexander Grund
2021-04-21 13:37 ` J. Bruce Fields
2021-04-21 13:49 ` Leon Romanovsky
2021-04-21 13:56 ` J. Bruce Fields
2021-04-22 19:39 ` J. Bruce Fields
2021-04-23 17:25 ` Leon Romanovsky
2021-04-23 18:07 ` J. Bruce Fields
2021-04-23 19:29 ` Leon Romanovsky
2021-04-23 21:48 ` J. Bruce Fields
2021-04-24 7:21 ` Leon Romanovsky
2021-04-24 18:34 ` Al Viro
2021-04-24 21:34 ` J. Bruce Fields
2021-04-25 0:41 ` Theodore Ts'o
2021-04-25 6:29 ` Greg KH
[not found] ` <20210426133605.GD21222@fieldses.org>
2021-04-26 13:47 ` J. Bruce Fields
2021-04-22 8:10 ` Sudip Mukherjee
2021-04-22 8:27 ` Greg KH
2021-04-21 12:51 ` Anna Schumaker
2021-04-21 14:15 ` Leon Romanovsky
2021-04-21 15:48 ` Theodore Ts'o
2021-04-21 17:34 ` Mike Rapoport
2021-04-22 3:57 ` Leon Romanovsky
2021-04-21 22:52 ` Guenter Roeck
[not found]
2021-04-21 19:49 ` Theodore Ts'o
2021-04-22 7:50 ` Eric Biggers
2021-04-21 20:27 Weikeng Chen
```
* * *
```
Reply instructions:
You may reply publicly to this message via plain-text email
using any one of the following methods:
* Save the following mbox file, import it into your mail client,
and reply-to-all from there: mbox
Avoid top-posting and favor interleaved quoting:
https://en.wikipedia.org/wiki/Posting_style#Interleaved_style
* Reply using the --to, --cc, and --in-reply-to
switches of git-send-email(1):
git send-email \
--in-reply-to=YH+zwQgBBGUJdiVK@unreal \
--to=leon@kernel.org \
--cc=anna.schumaker@netapp.com \
--cc=bfields@fieldses.org \
--cc=chuck.lever@oracle.com \
--cc=davem@davemloft.net \
--cc=dwysocha@redhat.com \
--cc=gregkh@linuxfoundation.org \
--cc=kuba@kernel.org \
--cc=linux-kernel@vger.kernel.org \
--cc=linux-nfs@vger.kernel.org \
--cc=netdev@vger.kernel.org \
--cc=pakki001@umn.edu \
--cc=trond.myklebust@hammerspace.com \
--subject='Re: [PATCH] SUNRPC: Add a check for gss_release_msg' \
/path/to/YOUR_REPLY
https://kernel.org/pub/software/scm/git/docs/git-send-email.html
* If your mail client supports setting the In-Reply-To header
via mailto: links, try the mailto: link
```
* * *
```
This is a public inbox, see mirroring instructions
for how to clone and mirror all data and code used for this inbox;
as well as URLs for NNTP newsgroup(s).
```
* * *
Here's an example of an XML-RPC request:
```xml
POST /RPC2 HTTP/1.0
User-Agent: Frontier/5.1.2 (WinNT)
Host: betty.userland.com
Content-Type: text/xml
Content-length: 181
examples.getStateName41
```
* * *
For example, running:
```
octosql "SELECT email, COUNT(*) as invoice_count
FROM invoices.csv JOIN mydb.customers ON invoices.customer_id = customers.id
WHERE first_name <= 'D'
GROUP BY email
ORDER BY invoice_count DESC" --explain 1
```
* * *
If you really want this functionality, you can implement it by using the
following function:
```
function sleepSync(timeout) {
const sab = new SharedArrayBuffer(1024);
const int32 = new Int32Array(sab);
Atomics.wait(int32, 0, 0, timeout);
}
```
* * *
It doesn’t get much simpler than this. You’ll notice we only define a main()
function followed by a println to stdout.
``` rust
fn main() {
println!("Hello, world!");
}
```
* * *
```ruby
# Ruby knows what you
# mean, even if you
# want to do math on
# an entire Array
cities = %w[ London
Oslo
Paris
Amsterdam
Berlin ]
visited = %w[Berlin Oslo]
puts "I still need " +
"to visit the " +
"following cities:",
cities - visited
```
* * *
The equivalent portion is just these two lines:
```
var btn = new Gtk.Button.with_label("Hello World");
btn.click.connect(win.close);
```
* * *
To preserve linebreaks inside a `div` using CSS:
```css hljs
div.code {
white-space: pre;
}
```
* * *
```css
@font-face {
src: url(http://lea.verou.me/logo.otf);
font-family: 'LeaVerou';
}
```
* * *
```none
This raw text
is not highlighted
but it still has
line numbers
```
* * *
Live JSX Editor JSX?
```
class HelloMessage extends React.Component {
render() {
return
Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus. Vestibulum id ligula porta felis euismod semper.
More
Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus. Vestibulum id ligula porta felis euismod semper.
More
output.default.golden 0000664 0000000 0000000 00000001104 14624642056 0041542 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/text_with_whitespace # Aktuelles
[Event Title](http://example.com/img.jpg "Event Title")
25 Mai
### [Event Title](http://example.com/event)
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. ...
[More](http://example.com/event1)
* * *
# Aktuelles
### [Title](http://example.com/some_url)
Fusce dapibus, tellus ac cursus commodo, tortor mauris condimentum nibh, ut fermentum massa justo sit amet risus. Vestibulum id ligula porta felis euismod semper.
[More](http://example.com/other_url) golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/turndown_demo/0000775 0000000 0000000 00000000000 14624642056 0034101 5 ustar 00root root 0000000 0000000 goldmark.golden 0000664 0000000 0000000 00000001503 14624642056 0037013 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/turndown_demo
Turndown Demo
This demonstrates turndown – an HTML to Markdown converter in JavaScript.
Usage
var turndownService = new TurndownService()
console.log(
turndownService.turndown('<h1>Hello world</h1>')
)
It aims to be CommonMark
compliant, and includes options to style the output. These options include:
output.default.golden 0000664 0000000 0000000 00000000445 14624642056 0036446 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestRealWorld/snippets/tweet [Max Mustermann @max\
\
Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. \
\
\
June 12th 2020\
\
17 Retweets93 Likes](http://example.com/status/1) golang-github-johanneskaufmann-html-to-markdown-1.6.0/utils.go 0000664 0000000 0000000 00000030370 14624642056 0024476 0 ustar 00root root 0000000 0000000 package md
import (
"bytes"
"fmt"
"regexp"
"strconv"
"strings"
"unicode"
"unicode/utf8"
"github.com/PuerkitoBio/goquery"
"golang.org/x/net/html"
)
/*
WARNING: The functions from this file can be used externally
but there is no garanty that they will stay exported.
*/
// CollectText returns the text of the node and all its children
func CollectText(n *html.Node) string {
text := &bytes.Buffer{}
collectText(n, text)
return text.String()
}
func collectText(n *html.Node, buf *bytes.Buffer) {
if n.Type == html.TextNode {
buf.WriteString(n.Data)
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
collectText(c, buf)
}
}
func getName(node *html.Node) string {
selec := &goquery.Selection{Nodes: []*html.Node{node}}
return goquery.NodeName(selec)
}
// What elements automatically trim their content?
// Don't add another space if the other element is going to add a
// space already.
func isTrimmedElement(name string) bool {
nodes := []string{
"a",
"strong", "b",
"i", "em",
"del", "s", "strike",
"code",
}
for _, node := range nodes {
if name == node {
return true
}
}
return false
}
func getPrevNodeText(node *html.Node) (string, bool) {
if node == nil {
return "", false
}
for ; node != nil; node = node.PrevSibling {
text := CollectText(node)
name := getName(node)
if name == "br" {
return "\n", true
}
// if the content is empty, try our luck with the next node
if strings.TrimSpace(text) == "" {
continue
}
if isTrimmedElement(name) {
text = strings.TrimSpace(text)
}
return text, true
}
return "", false
}
func getNextNodeText(node *html.Node) (string, bool) {
if node == nil {
return "", false
}
for ; node != nil; node = node.NextSibling {
text := CollectText(node)
name := getName(node)
if name == "br" {
return "\n", true
}
// if the content is empty, try our luck with the next node
if strings.TrimSpace(text) == "" {
continue
}
// if you have "a a a", three elements that are trimmed, then only add
// a space to one side, since the other's are also adding a space.
if isTrimmedElement(name) {
text = " "
}
return text, true
}
return "", false
}
// AddSpaceIfNessesary adds spaces to the text based on the neighbors.
// That makes sure that there is always a space to the side, to recognize the delimiter.
func AddSpaceIfNessesary(selec *goquery.Selection, markdown string) string {
if len(selec.Nodes) == 0 {
return markdown
}
rootNode := selec.Nodes[0]
prev, hasPrev := getPrevNodeText(rootNode.PrevSibling)
if hasPrev {
lastChar, size := utf8.DecodeLastRuneInString(prev)
if size > 0 && !unicode.IsSpace(lastChar) {
markdown = " " + markdown
}
}
next, hasNext := getNextNodeText(rootNode.NextSibling)
if hasNext {
firstChar, size := utf8.DecodeRuneInString(next)
if size > 0 && !unicode.IsSpace(firstChar) && !unicode.IsPunct(firstChar) {
markdown = markdown + " "
}
}
return markdown
}
func isLineCodeDelimiter(chars []rune) bool {
if len(chars) < 3 {
return false
}
// TODO: If it starts with 4 (instead of 3) fence characters, we should only end it
// if we see the same amount of ending fence characters.
return chars[0] == '`' && chars[1] == '`' && chars[2] == '`'
}
// TrimpLeadingSpaces removes spaces from the beginning of a line
// but makes sure that list items and code blocks are not affected.
func TrimpLeadingSpaces(text string) string {
var insideCodeBlock bool
lines := strings.Split(text, "\n")
for index := range lines {
chars := []rune(lines[index])
if isLineCodeDelimiter(chars) {
if !insideCodeBlock {
// start the code block
insideCodeBlock = true
} else {
// end the code block
insideCodeBlock = false
}
}
if insideCodeBlock {
// We are inside a code block and don't want to
// disturb that formatting (e.g. python indentation)
continue
}
var spaces int
for i := 0; i < len(chars); i++ {
if unicode.IsSpace(chars[i]) {
if chars[i] == ' ' {
spaces = spaces + 4
} else {
spaces++
}
continue
}
// this seems to be a list item
if chars[i] == '-' {
break
}
// this seems to be a code block
if spaces >= 4 {
break
}
// remove the space characters from the string
chars = chars[i:]
break
}
lines[index] = string(chars)
}
return strings.Join(lines, "\n")
}
// TrimTrailingSpaces removes unnecessary spaces from the end of lines.
func TrimTrailingSpaces(text string) string {
parts := strings.Split(text, "\n")
for i := range parts {
parts[i] = strings.TrimRightFunc(parts[i], func(r rune) bool {
return unicode.IsSpace(r)
})
}
return strings.Join(parts, "\n")
}
// The same as `multipleNewLinesRegex`, but applies to escaped new lines inside a link `\n\`
var multipleNewLinesInLinkRegex = regexp.MustCompile(`(\n\\){1,}`) // `([\n\r\s]\\)`
// EscapeMultiLine deals with multiline content inside a link
func EscapeMultiLine(content string) string {
content = strings.TrimSpace(content)
content = strings.Replace(content, "\n", `\`+"\n", -1)
content = multipleNewLinesInLinkRegex.ReplaceAllString(content, "\n\\")
return content
}
func calculateCodeFenceOccurrences(fenceChar rune, content string) int {
var occurrences []int
var charsTogether int
for _, char := range content {
// we encountered a fence character, now count how many
// are directly afterwards
if char == fenceChar {
charsTogether++
} else if charsTogether != 0 {
occurrences = append(occurrences, charsTogether)
charsTogether = 0
}
}
// if the last element in the content was a fenceChar
if charsTogether != 0 {
occurrences = append(occurrences, charsTogether)
}
return findMax(occurrences)
}
// CalculateCodeFence can be passed the content of a code block and it returns
// how many fence characters (` or ~) should be used.
//
// This is useful if the html content includes the same fence characters
// for example ```
// -> https://stackoverflow.com/a/49268657
func CalculateCodeFence(fenceChar rune, content string) string {
repeat := calculateCodeFenceOccurrences(fenceChar, content)
// the outer fence block always has to have
// at least one character more than any content inside
repeat++
// you have to have at least three fence characters
// to be recognized as a code block
if repeat < 3 {
repeat = 3
}
return strings.Repeat(string(fenceChar), repeat)
}
func findMax(a []int) (max int) {
for i, value := range a {
if i == 0 {
max = a[i]
}
if value > max {
max = value
}
}
return max
}
func getCodeWithoutTags(startNode *html.Node) []byte {
var buf bytes.Buffer
var f func(*html.Node)
f = func(n *html.Node) {
if n.Type == html.ElementNode && (n.Data == "style" || n.Data == "script" || n.Data == "textarea") {
return
}
if n.Type == html.ElementNode && (n.Data == "br" || n.Data == "div") {
buf.WriteString("\n")
}
if n.Type == html.TextNode {
buf.WriteString(n.Data)
return
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
f(c)
}
}
f(startNode)
return buf.Bytes()
}
// getCodeContent gets the content of pre/code and unescapes the encoded characters.
// Returns "" if there is an error.
func getCodeContent(selec *goquery.Selection) string {
if len(selec.Nodes) == 0 {
return ""
}
code := getCodeWithoutTags(selec.Nodes[0])
return string(code)
}
// delimiterForEveryLine puts the delimiter not just at the start and end of the string
// but if the text is divided on multiple lines, puts the delimiters on every line with content.
//
// Otherwise the bold/italic delimiters won't be recognized if it contains new line characters.
func delimiterForEveryLine(text string, delimiter string) string {
lines := strings.Split(text, "\n")
for i, line := range lines {
line = strings.TrimSpace(line)
if line == "" {
// Skip empty lines
continue
}
lines[i] = delimiter + line + delimiter
}
return strings.Join(lines, "\n")
}
// isWrapperListItem returns wether the list item has own
// content or is just a wrapper for another list.
// e.g. "
..."
func isWrapperListItem(s *goquery.Selection) bool {
directText := s.Contents().Not("ul").Not("ol").Text()
noOwnText := strings.TrimSpace(directText) == ""
childIsList := s.ChildrenFiltered("ul").Length() > 0 || s.ChildrenFiltered("ol").Length() > 0
return noOwnText && childIsList
}
// getListStart returns the integer from which the counting
// for for the list items should start from.
// -> https://developer.mozilla.org/en-US/docs/Web/HTML/Element/ol#start
func getListStart(parent *goquery.Selection) int {
val := parent.AttrOr("start", "")
if val == "" {
return 1
}
num, err := strconv.Atoi(val)
if err != nil {
return 1
}
if num < 0 {
return 1
}
return num
}
// getListPrefix returns the appropriate prefix for the list item.
// For example "- ", "* ", "1. ", "01. ", ...
func getListPrefix(opt *Options, s *goquery.Selection) string {
if isWrapperListItem(s) {
return ""
}
parent := s.Parent()
if parent.Is("ul") {
return opt.BulletListMarker + " "
} else if parent.Is("ol") {
start := getListStart(parent)
currentIndex := start + s.Index()
lastIndex := parent.Children().Last().Index() + 1
maxLength := len(strconv.Itoa(lastIndex))
// pad the numbers so that all prefix numbers in the list take up the same space
// `%02d.` -> "01. "
format := `%0` + strconv.Itoa(maxLength) + `d. `
return fmt.Sprintf(format, currentIndex)
}
// If the HTML is malformed and the list element isn't in a ul or ol, return no prefix
return ""
}
// countListParents counts how much space is reserved for the prefixes at all the parent lists.
// This is useful to calculate the correct level of indentation for nested lists.
func countListParents(opt *Options, selec *goquery.Selection) (int, int) {
var values []int
for n := selec.Parent(); n != nil; n = n.Parent() {
if n.Is("li") {
continue
}
if !n.Is("ul") && !n.Is("ol") {
break
}
prefix := n.Children().First().AttrOr(attrListPrefix, "")
values = append(values, len(prefix))
}
// how many spaces are reserved for the prefixes of my siblings
var prefixCount int
// how many spaces are reserved in total for all of the other
// list parents up the tree
var previousPrefixCounts int
for i, val := range values {
if i == 0 {
prefixCount = val
continue
}
previousPrefixCounts += val
}
return prefixCount, previousPrefixCounts
}
// IndentMultiLineListItem makes sure that multiline list items
// are properly indented.
func IndentMultiLineListItem(opt *Options, text string, spaces int) string {
parts := strings.Split(text, "\n")
for i := range parts {
// dont touch the first line since its indented through the prefix
if i == 0 {
continue
}
if isListItem(opt, parts[i]) {
return strings.Join(parts, "\n")
}
indent := strings.Repeat(" ", spaces)
parts[i] = indent + parts[i]
}
return strings.Join(parts, "\n")
}
// isListItem checks wether the line is a markdown list item
func isListItem(opt *Options, line string) bool {
b := []rune(line)
bulletMarker := []rune(opt.BulletListMarker)[0]
var hasNumber bool
var hasMarker bool
var hasSpace bool
for i := 0; i < len(b); i++ {
// A marker followed by a space qualifies as a list item
if hasMarker && hasSpace {
if b[i] == bulletMarker {
// But if another BulletListMarker is found, it
// might be a HorizontalRule
return false
}
if !unicode.IsSpace(b[i]) {
// Now we have some text
return true
}
}
if hasMarker {
if unicode.IsSpace(b[i]) {
hasSpace = true
continue
}
// A marker like "1." that is not immediately followed by a space
// is probably a false positive
return false
}
if b[i] == bulletMarker {
hasMarker = true
continue
}
if hasNumber && b[i] == '.' {
hasMarker = true
continue
}
if unicode.IsDigit(b[i]) {
hasNumber = true
continue
}
if unicode.IsSpace(b[i]) {
continue
}
// If we encouter any other character
// before finding an indicator, its
// not a list item
return false
}
return false
}
// IndexWithText is similar to goquery's Index function but
// returns the index of the current element while
// NOT counting the empty elements beforehand.
func IndexWithText(s *goquery.Selection) int {
return s.PrevAll().FilterFunction(func(i int, s *goquery.Selection) bool {
return strings.TrimSpace(s.Text()) != ""
}).Length()
}
golang-github-johanneskaufmann-html-to-markdown-1.6.0/utils_test.go 0000664 0000000 0000000 00000023551 14624642056 0025540 0 ustar 00root root 0000000 0000000 package md
import (
"strings"
"testing"
"github.com/PuerkitoBio/goquery"
"golang.org/x/net/html"
)
func getNodeFromString(t *testing.T, rawHTML string) *html.Node {
docNode, err := html.Parse(strings.NewReader(rawHTML))
if err != nil {
t.Error(err)
return nil
}
// -> #document -> body -> actuall content
return docNode.FirstChild.LastChild.FirstChild
}
func TestAddSpaceIfNessesary(t *testing.T) {
var tests = []struct {
Name string
Prev string
Next string
Markdown string
Expect string
}{
{
Name: "dont count comment",
Prev: `
`,
Next: `
`,
Markdown: `_Comment Content_`,
Expect: `_Comment Content_`,
},
{
Name: "bold with break",
Prev: ` `,
Next: ` `,
Markdown: `**Bold**`,
Expect: `**Bold**`,
},
{
Name: "italic with no space",
Prev: ``,
Next: `and no space afterward.`, // #text
Markdown: `_Content_`,
Expect: `_Content_ `,
},
{
Name: "bold with no space",
Prev: `Some`,
Next: `Text`,
Markdown: `**Bold**`,
Expect: ` **Bold** `,
},
{
Name: "bold with no space in span",
Prev: `Some`,
Next: `Text`,
Markdown: `**Bold**`,
Expect: ` **Bold** `,
},
{
Name: "italic with no space",
Prev: ``,
Next: `and no space afterward.`,
Markdown: `_Content_`,
Expect: `_Content_ `,
},
{
Name: "github example without new lines",
Prev: `go`,
Next: `html`,
Markdown: `[golang](http://example.com/topics/golang "Topic: golang")`,
Expect: ` [golang](http://example.com/topics/golang "Topic: golang")`,
},
{
Name: "github example",
Prev: `
go
`,
Next: `
html
`,
Markdown: `[golang](http://example.com/topics/golang "Topic: golang")`,
Expect: ` [golang](http://example.com/topics/golang "Topic: golang")`,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
// build a selection for goquery with siblings
selec := &goquery.Selection{
Nodes: []*html.Node{
{
Data: "a",
PrevSibling: getNodeFromString(t, test.Prev),
NextSibling: getNodeFromString(t, test.Next),
},
},
}
output := AddSpaceIfNessesary(selec, test.Markdown)
if output != test.Expect {
t.Errorf("expected '%s' but got '%s'", test.Expect, output)
}
})
}
}
func TestTrimpLeadingSpaces(t *testing.T) {
var tests = []struct {
Name string
Text string
Expect string
}{
{
Name: "trim normal text",
Text: `
This is a normal paragraph
this as well
just with some spaces before
`,
Expect: `
This is a normal paragraph
this as well
just with some spaces before
`,
},
{
Name: "dont trim nested lists",
Text: `
- Home
- About
- People
- History
- 2019
- 2020
`,
Expect: `
- Home
- About
- People
- History
- 2019
- 2020
`,
},
{
Name: "dont trim list with multiple paragraphs",
Text: `
1. This is a list item with two paragraphs. Lorem ipsum dolor
sit amet, consectetuer adipiscing elit. Aliquam hendrerit
mi posuere lectus.
Vestibulum enim wisi, viverra nec, fringilla in, laoreet
vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
sit amet velit.
2. Suspendisse id sem consectetuer libero luctus adipiscing.
`,
Expect: `
1. This is a list item with two paragraphs. Lorem ipsum dolor
sit amet, consectetuer adipiscing elit. Aliquam hendrerit
mi posuere lectus.
Vestibulum enim wisi, viverra nec, fringilla in, laoreet
vitae, risus. Donec sit amet nisl. Aliquam semper ipsum
sit amet velit.
2. Suspendisse id sem consectetuer libero luctus adipiscing.
`,
},
{
Name: "dont trim code blocks",
Text: `
This is a normal paragraph:
This is a code block.
`,
Expect: `
This is a normal paragraph:
This is a code block.
`,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
output := TrimpLeadingSpaces(test.Text)
if output != test.Expect {
t.Errorf("expected '%s' but got '%s'", test.Expect, output)
}
})
}
}
func TestTrimTrailingSpaces(t *testing.T) {
var tests = []struct {
Name string
Text string
Expect string
}{
{
Name: "trim after normal text",
Text: `
1\. xxx
2\. xxxx
`,
Expect: `
1\. xxx
2\. xxxx
`,
},
{
Name: "dont trim inside normal text",
Text: "When `x = 3`, that means `x + 2 = 5`",
Expect: "When `x = 3`, that means `x + 2 = 5`",
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
output := TrimTrailingSpaces(test.Text)
if output != test.Expect {
t.Errorf("expected '%s' but got '%s'", test.Expect, output)
}
})
}
}
func TestEscapeMultiLine(t *testing.T) {
var tests = []struct {
Name string
Text string
Expect string
}{
{
Name: "escape new lines",
Text: `line1
line2
line3
line4`,
Expect: `line1\
line2\
\
line3\
\
line4`,
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
output := EscapeMultiLine(test.Text)
if output != test.Expect {
t.Errorf("expected '%s' but got '%s'", test.Expect, output)
}
})
}
}
func TestCalculateCodeFence(t *testing.T) {
var tests = []struct {
Name string
FenceChar rune
Text string
Expect string
}{
{
Name: "no occurrences with backtick",
FenceChar: '`',
Text: `normal ~~~ code block`,
Expect: "```",
},
{
Name: "no occurrences with tilde",
FenceChar: '~',
Text: "normal ``` code block",
Expect: "~~~",
},
{
Name: "one exact occurrence",
FenceChar: '`',
Text: "```",
Expect: "````",
},
{
Name: "one occurrences with backtick",
FenceChar: '`',
Text: "normal ``` code block",
Expect: "````",
},
{
Name: "one bigger occurrences with backtick",
FenceChar: '`',
Text: "normal ````` code block",
Expect: "``````",
},
{
Name: "multiple occurrences with backtick",
FenceChar: '`',
Text: "normal ``` code `````` block",
Expect: "```````",
},
{
Name: "multiple occurrences with tilde",
FenceChar: '~',
Text: "normal ~~~ code ~~~~~~~~~~~~ block",
Expect: "~~~~~~~~~~~~~",
},
{
Name: "multiple occurrences on different lines with tilde",
FenceChar: '~',
Text: `
normal
~~~
code ~~~~~~~~~~~~ block
`,
Expect: "~~~~~~~~~~~~~",
},
}
for _, test := range tests {
t.Run(test.Name, func(t *testing.T) {
output := CalculateCodeFence(test.FenceChar, test.Text)
if output != test.Expect {
t.Errorf("expected '%s' (x%d) but got '%s' (x%d)", test.Expect, strings.Count(test.Expect, string(test.FenceChar)), output, strings.Count(output, string(test.FenceChar)))
}
})
}
}
func TestIsListItem(t *testing.T) {
var tests = []struct {
name string
opt *Options
line string
expected bool
}{
{
name: "nothing",
opt: &Options{
BulletListMarker: "-",
},
line: "",
expected: false,
},
{
name: "just spaces",
opt: &Options{
BulletListMarker: "-",
},
line: " ",
expected: false,
},
{
name: "just text",
opt: &Options{
BulletListMarker: "-",
},
line: " text",
expected: false,
},
{
name: "just numbers",
opt: &Options{
BulletListMarker: "-",
},
line: " 123",
expected: false,
},
{
name: "unordered: with -",
opt: &Options{
BulletListMarker: "-",
},
line: " - item",
expected: true,
},
{
name: "unordered: with *",
opt: &Options{
BulletListMarker: "*",
},
line: " * item",
expected: true,
},
{
name: "unordered: with * false positive",
opt: &Options{
BulletListMarker: "*",
},
line: " - item",
expected: false,
},
{
name: "unordered: with multiple spaces",
opt: &Options{
BulletListMarker: "-",
},
line: " - item",
expected: true,
},
{
name: "unordered: without space",
opt: &Options{
BulletListMarker: "-",
},
line: " -item",
expected: false,
},
{
name: "ordered: without space",
opt: &Options{
BulletListMarker: "-",
},
line: " 1.item",
expected: false,
},
{
name: "ordered: with space",
opt: &Options{
BulletListMarker: "-",
},
line: " 1. item",
expected: true,
},
{
name: "ordered: without dot",
opt: &Options{
BulletListMarker: "-",
},
line: " 1 item",
expected: false,
},
{
name: "ordered: with dot before",
opt: &Options{
BulletListMarker: "-",
},
line: " .1 item",
expected: false,
},
{
name: "ordered: with big number",
opt: &Options{
BulletListMarker: "-",
},
line: " 1001. item",
expected: true,
},
{
name: "ordered: with date",
opt: &Options{
BulletListMarker: "-",
},
line: " 01.01 January",
expected: false,
},
{
name: "with divider",
opt: &Options{
BulletListMarker: "*",
},
line: "***",
expected: false,
},
{
name: "with divider and spaces",
opt: &Options{
BulletListMarker: "*",
},
line: "* * *",
expected: false,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
result := isListItem(test.opt, test.line)
if result != test.expected {
t.Errorf("expected '%+v' but got '%+v' for '%s'", test.expected, result, test.line)
}
})
}
}
output.default.golden 0000664 0000000 0000000 00000000043 14624642056 0037007 0 ustar 00root root 0000000 0000000 golang-github-johanneskaufmann-html-to-markdown-1.6.0/testdata/TestCommonmark/keep_remove_tag 
Max Mustermann @max
<pre>is it modifies whitespace processing as well: all spaces are preserved, and wrapping is switched off. Unless there's a way to switch this off?