pax_global_header 0000666 0000000 0000000 00000000064 14117006534 0014513 g ustar 00root root 0000000 0000000 52 comment=c57ea9efcad0d4e6d75d0430b8dcbd9cda297ca1
parse-2.5.21/ 0000775 0000000 0000000 00000000000 14117006534 0012714 5 ustar 00root root 0000000 0000000 parse-2.5.21/.gitattributes 0000664 0000000 0000000 00000000044 14117006534 0015605 0 ustar 00root root 0000000 0000000 tests/*/corpus/* linguist-generated
parse-2.5.21/.github/ 0000775 0000000 0000000 00000000000 14117006534 0014254 5 ustar 00root root 0000000 0000000 parse-2.5.21/.github/dependabot.yml 0000664 0000000 0000000 00000000136 14117006534 0017104 0 ustar 00root root 0000000 0000000 version: 2
updates:
- package-ecosystem: gomod
directory: /
schedule:
interval: daily
parse-2.5.21/.github/workflows/ 0000775 0000000 0000000 00000000000 14117006534 0016311 5 ustar 00root root 0000000 0000000 parse-2.5.21/.github/workflows/go.yml 0000664 0000000 0000000 00000001442 14117006534 0017442 0 ustar 00root root 0000000 0000000 name: Go
on:
push:
branches:
- master
pull_request:
branches:
- master
jobs:
build:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Go
uses: actions/setup-go@v2
with:
go-version: 1.16
- name: Build
run: go build -v ./...
- name: Tests with coverage
run: go test -race -v -count=1 -coverprofile=coverage.out ./...
- name: Upload Code Coverage
uses: codecov/codecov-action@v1
with:
name: codecov
fail_ci_if_error: true
golangci:
name: lint
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: golangci-lint
uses: golangci/golangci-lint-action@v2
continue-on-error: true
with:
version: latest
parse-2.5.21/.gitignore 0000664 0000000 0000000 00000000141 14117006534 0014700 0 ustar 00root root 0000000 0000000 tests/*/fuzz-fuzz.zip
tests/*/crashers
tests/*/suppressions
tests/*/corpus/*
!tests/*/corpus/*.*
parse-2.5.21/.golangci.yml 0000664 0000000 0000000 00000000303 14117006534 0015274 0 ustar 00root root 0000000 0000000 linters:
enable:
- depguard
- dogsled
- gofmt
- goimports
- golint
- gosec
- govet
- megacheck
- misspell
- nakedret
- prealloc
- unconvert
- unparam
- wastedassign
parse-2.5.21/LICENSE.md 0000664 0000000 0000000 00000002062 14117006534 0014320 0 ustar 00root root 0000000 0000000 Copyright (c) 2015 Taco de Wolff
Permission is hereby granted, free of charge, to any person
obtaining a copy of this software and associated documentation
files (the "Software"), to deal in the Software without
restriction, including without limitation the rights to use,
copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the
Software is furnished to do so, subject to the following
conditions:
The above copyright notice and this permission notice shall be
included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE. parse-2.5.21/README.md 0000664 0000000 0000000 00000012554 14117006534 0014202 0 ustar 00root root 0000000 0000000 # Parse [](https://pkg.go.dev/github.com/tdewolff/parse/v2?tab=doc) [](https://goreportcard.com/report/github.com/tdewolff/parse) [](https://coveralls.io/github/tdewolff/parse?branch=master) [](https://www.patreon.com/tdewolff)
This package contains several lexers and parsers written in [Go][1]. All subpackages are built to be streaming, high performance and to be in accordance with the official (latest) specifications.
The lexers are implemented using `buffer.Lexer` in https://github.com/tdewolff/parse/buffer and the parsers work on top of the lexers. Some subpackages have hashes defined (using [Hasher](https://github.com/tdewolff/hasher)) that speed up common byte-slice comparisons.
## Buffer
### Reader
Reader is a wrapper around a `[]byte` that implements the `io.Reader` interface. It is comparable to `bytes.Reader` but has slightly different semantics (and a slightly smaller memory footprint).
### Writer
Writer is a buffer that implements the `io.Writer` interface and expands the buffer as needed. The reset functionality allows for better memory reuse. After calling `Reset`, it will overwrite the current buffer and thus reduce allocations.
### Lexer
Lexer is a read buffer specifically designed for building lexers. It keeps track of two positions: a start and end position. The start position is the beginning of the current token being parsed, the end position is being moved forward until a valid token is found. Calling `Shift` will collapse the positions to the end and return the parsed `[]byte`.
Moving the end position can go through `Move(int)` which also accepts negative integers. One can also use `Pos() int` to try and parse a token, and if it fails rewind with `Rewind(int)`, passing the previously saved position.
`Peek(int) byte` will peek forward (relative to the end position) and return the byte at that location. `PeekRune(int) (rune, int)` returns UTF-8 runes and its length at the given **byte** position. Upon an error `Peek` will return `0`, the **user must peek at every character** and not skip any, otherwise it may skip a `0` and panic on out-of-bounds indexing.
`Lexeme() []byte` will return the currently selected bytes, `Skip()` will collapse the selection. `Shift() []byte` is a combination of `Lexeme() []byte` and `Skip()`.
When the passed `io.Reader` returned an error, `Err() error` will return that error even if not at the end of the buffer.
### StreamLexer
StreamLexer behaves like Lexer but uses a buffer pool to read in chunks from `io.Reader`, retaining old buffers in memory that are still in use, and re-using old buffers otherwise. Calling `Free(n int)` frees up `n` bytes from the internal buffer(s). It holds an array of buffers to accommodate for keeping everything in-memory. Calling `ShiftLen() int` returns the number of bytes that have been shifted since the previous call to `ShiftLen`, which can be used to specify how many bytes need to be freed up from the buffer. If you don't need to keep returned byte slices around, call `Free(ShiftLen())` after every `Shift` call.
## Strconv
This package contains string conversion function much like the standard library's `strconv` package, but it is specifically tailored for the performance needs within the `minify` package.
For example, the floating-point to string conversion function is approximately twice as fast as the standard library, but it is not as precise.
## CSS
This package is a CSS3 lexer and parser. Both follow the specification at [CSS Syntax Module Level 3](http://www.w3.org/TR/css-syntax-3/). The lexer takes an io.Reader and converts it into tokens until the EOF. The parser returns a parse tree of the full io.Reader input stream, but the low-level `Next` function can be used for stream parsing to returns grammar units until the EOF.
[See README here](https://github.com/tdewolff/parse/tree/master/css).
## HTML
This package is an HTML5 lexer. It follows the specification at [The HTML syntax](http://www.w3.org/TR/html5/syntax.html). The lexer takes an io.Reader and converts it into tokens until the EOF.
[See README here](https://github.com/tdewolff/parse/tree/master/html).
## JS
This package is a JS lexer (ECMA-262, edition 6.0). It follows the specification at [ECMAScript Language Specification](http://www.ecma-international.org/ecma-262/6.0/). The lexer takes an io.Reader and converts it into tokens until the EOF.
[See README here](https://github.com/tdewolff/parse/tree/master/js).
## JSON
This package is a JSON parser (ECMA-404). It follows the specification at [JSON](http://json.org/). The parser takes an io.Reader and converts it into tokens until the EOF.
[See README here](https://github.com/tdewolff/parse/tree/master/json).
## SVG
This package contains common hashes for SVG1.1 tags and attributes.
## XML
This package is an XML1.0 lexer. It follows the specification at [Extensible Markup Language (XML) 1.0 (Fifth Edition)](http://www.w3.org/TR/xml/). The lexer takes an io.Reader and converts it into tokens until the EOF.
[See README here](https://github.com/tdewolff/parse/tree/master/xml).
## License
Released under the [MIT license](LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/buffer/ 0000775 0000000 0000000 00000000000 14117006534 0014165 5 ustar 00root root 0000000 0000000 parse-2.5.21/buffer/buffer.go 0000664 0000000 0000000 00000001573 14117006534 0015773 0 ustar 00root root 0000000 0000000 // Package buffer contains buffer and wrapper types for byte slices. It is useful for writing lexers or other high-performance byte slice handling.
// The `Reader` and `Writer` types implement the `io.Reader` and `io.Writer` respectively and provide a thinner and faster interface than `bytes.Buffer`.
// The `Lexer` type is useful for building lexers because it keeps track of the start and end position of a byte selection, and shifts the bytes whenever a valid token is found.
// The `StreamLexer` does the same, but keeps a buffer pool so that it reads a limited amount at a time, allowing to parse from streaming sources.
package buffer
// defaultBufSize specifies the default initial length of internal buffers.
var defaultBufSize = 4096
// MinBuf specifies the default initial length of internal buffers.
// Solely here to support old versions of parse.
var MinBuf = defaultBufSize
parse-2.5.21/buffer/lexer.go 0000664 0000000 0000000 00000007471 14117006534 0015644 0 ustar 00root root 0000000 0000000 package buffer
import (
"io"
"io/ioutil"
)
var nullBuffer = []byte{0}
// Lexer is a buffered reader that allows peeking forward and shifting, taking an io.Reader.
// It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.
type Lexer struct {
buf []byte
pos int // index in buf
start int // index in buf
err error
restore func()
}
// NewLexer returns a new Lexer for a given io.Reader, and uses ioutil.ReadAll to read it into a byte slice.
// If the io.Reader implements Bytes, that is used instead.
// It will append a NULL at the end of the buffer.
func NewLexer(r io.Reader) *Lexer {
var b []byte
if r != nil {
if buffer, ok := r.(interface {
Bytes() []byte
}); ok {
b = buffer.Bytes()
} else {
var err error
b, err = ioutil.ReadAll(r)
if err != nil {
return &Lexer{
buf: nullBuffer,
err: err,
}
}
}
}
return NewLexerBytes(b)
}
// NewLexerBytes returns a new Lexer for a given byte slice, and appends NULL at the end.
// To avoid reallocation, make sure the capacity has room for one more byte.
func NewLexerBytes(b []byte) *Lexer {
z := &Lexer{
buf: b,
}
n := len(b)
if n == 0 {
z.buf = nullBuffer
} else {
// Append NULL to buffer, but try to avoid reallocation
if cap(b) > n {
// Overwrite next byte but restore when done
b = b[:n+1]
c := b[n]
b[n] = 0
z.buf = b
z.restore = func() {
b[n] = c
}
} else {
z.buf = append(b, 0)
}
}
return z
}
// Restore restores the replaced byte past the end of the buffer by NULL.
func (z *Lexer) Restore() {
if z.restore != nil {
z.restore()
z.restore = nil
}
}
// Err returns the error returned from io.Reader or io.EOF when the end has been reached.
func (z *Lexer) Err() error {
return z.PeekErr(0)
}
// PeekErr returns the error at position pos. When pos is zero, this is the same as calling Err().
func (z *Lexer) PeekErr(pos int) error {
if z.err != nil {
return z.err
} else if z.pos+pos >= len(z.buf)-1 {
return io.EOF
}
return nil
}
// Peek returns the ith byte relative to the end position.
// Peek returns 0 when an error has occurred, Err returns the error.
func (z *Lexer) Peek(pos int) byte {
pos += z.pos
return z.buf[pos]
}
// PeekRune returns the rune and rune length of the ith byte relative to the end position.
func (z *Lexer) PeekRune(pos int) (rune, int) {
// from unicode/utf8
c := z.Peek(pos)
if c < 0xC0 || z.Peek(pos+1) == 0 {
return rune(c), 1
} else if c < 0xE0 || z.Peek(pos+2) == 0 {
return rune(c&0x1F)<<6 | rune(z.Peek(pos+1)&0x3F), 2
} else if c < 0xF0 || z.Peek(pos+3) == 0 {
return rune(c&0x0F)<<12 | rune(z.Peek(pos+1)&0x3F)<<6 | rune(z.Peek(pos+2)&0x3F), 3
}
return rune(c&0x07)<<18 | rune(z.Peek(pos+1)&0x3F)<<12 | rune(z.Peek(pos+2)&0x3F)<<6 | rune(z.Peek(pos+3)&0x3F), 4
}
// Move advances the position.
func (z *Lexer) Move(n int) {
z.pos += n
}
// Pos returns a mark to which can be rewinded.
func (z *Lexer) Pos() int {
return z.pos - z.start
}
// Rewind rewinds the position to the given position.
func (z *Lexer) Rewind(pos int) {
z.pos = z.start + pos
}
// Lexeme returns the bytes of the current selection.
func (z *Lexer) Lexeme() []byte {
return z.buf[z.start:z.pos:z.pos]
}
// Skip collapses the position to the end of the selection.
func (z *Lexer) Skip() {
z.start = z.pos
}
// Shift returns the bytes of the current selection and collapses the position to the end of the selection.
func (z *Lexer) Shift() []byte {
b := z.buf[z.start:z.pos:z.pos]
z.start = z.pos
return b
}
// Offset returns the character position in the buffer.
func (z *Lexer) Offset() int {
return z.pos
}
// Bytes returns the underlying buffer.
func (z *Lexer) Bytes() []byte {
return z.buf[: len(z.buf)-1 : len(z.buf)-1]
}
// Reset resets position to the underlying buffer.
func (z *Lexer) Reset() {
z.start = 0
z.pos = 0
}
parse-2.5.21/buffer/lexer_test.go 0000664 0000000 0000000 00000010105 14117006534 0016667 0 ustar 00root root 0000000 0000000 package buffer
import (
"bytes"
"io"
"testing"
"github.com/tdewolff/test"
)
func TestLexer(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewLexer(bytes.NewBufferString(s))
test.Bytes(t, z.Bytes(), []byte(s), "bytes match original buffer")
test.T(t, z.err, nil, "buffer has no error")
test.T(t, z.Err(), nil, "buffer is at EOF but must not return EOF until we reach that")
test.That(t, z.Pos() == 0, "buffer must start at position 0")
test.That(t, z.Peek(0) == 'L', "first character must be 'L'")
test.That(t, z.Peek(1) == 'o', "second character must be 'o'")
z.Move(1)
test.That(t, z.Peek(0) == 'o', "must be 'o' at position 1")
test.That(t, z.Peek(1) == 'r', "must be 'r' at position 1")
z.Rewind(6)
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 6")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 7")
test.T(t, z.Offset(), 6, "offset")
test.Bytes(t, z.Lexeme(), []byte("Lorem "), "buffered string must now read 'Lorem ' when at position 6")
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.Pos() == 0, "after shifting position must be 0")
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 0 after shifting")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 1 after shifting")
test.T(t, z.Err(), nil, "error must be nil at this point")
z.Move(len(s) - len("Lorem ") - 1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer")
z.Skip()
test.That(t, z.Pos() == 0, "after skipping position must be 0")
z.Move(1)
test.T(t, z.Err(), io.EOF, "error must be EOF when past the buffer")
z.Move(-1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer, even when it has been past the buffer")
z.Reset()
test.That(t, z.Peek(0) == 'L', "must be 'L' at position 0")
test.That(t, z.Peek(1) == 'o', "must be 'o' at position 1")
}
func TestLexerRunes(t *testing.T) {
z := NewLexer(bytes.NewBufferString("aæ†\U00100000"))
r, n := z.PeekRune(0)
test.That(t, n == 1, "first character must be length 1")
test.That(t, r == 'a', "first character must be rune 'a'")
r, n = z.PeekRune(1)
test.That(t, n == 2, "second character must be length 2")
test.That(t, r == 'æ', "second character must be rune 'æ'")
r, n = z.PeekRune(3)
test.That(t, n == 3, "fourth character must be length 3")
test.That(t, r == '†', "fourth character must be rune '†'")
r, n = z.PeekRune(6)
test.That(t, n == 4, "seventh character must be length 4")
test.That(t, r == '\U00100000', "seventh character must be rune '\U00100000'")
}
func TestLexerBadRune(t *testing.T) {
z := NewLexer(bytes.NewBufferString("\xF0")) // expect four byte rune
r, n := z.PeekRune(0)
test.T(t, n, 1, "length")
test.T(t, r, rune(0xF0), "rune")
}
func TestLexerZeroLen(t *testing.T) {
z := NewLexer(test.NewPlainReader(bytes.NewBufferString("")))
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.Bytes(t, z.Bytes(), []byte{}, "bytes match original buffer")
}
func TestLexerEmptyReader(t *testing.T) {
z := NewLexer(test.NewEmptyReader())
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), io.EOF, "error must be EOF")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
func TestLexerErrorReader(t *testing.T) {
z := NewLexer(test.NewErrorReader(0))
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), test.ErrPlain, "error must be ErrPlain")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
func TestLexerBytes(t *testing.T) {
b := []byte{'t', 'e', 's', 't'}
z := NewLexerBytes(b)
test.That(t, z.Peek(4) == 0, "fifth character must yield NULL")
}
func TestLexerRestore(t *testing.T) {
b := []byte{'a', 'b', 'c', 'd'}
z := NewLexerBytes(b[:2])
test.T(t, len(z.buf), 3, "must have terminating NULL")
test.T(t, z.buf[2], byte(0), "must have terminating NULL")
test.Bytes(t, b, []byte{'a', 'b', 0, 'd'}, "terminating NULL overwrites underlying buffer")
z.Restore()
test.Bytes(t, b, []byte{'a', 'b', 'c', 'd'}, "terminating NULL has been restored")
}
parse-2.5.21/buffer/reader.go 0000664 0000000 0000000 00000001556 14117006534 0015765 0 ustar 00root root 0000000 0000000 package buffer
import "io"
// Reader implements an io.Reader over a byte slice.
type Reader struct {
buf []byte
pos int
}
// NewReader returns a new Reader for a given byte slice.
func NewReader(buf []byte) *Reader {
return &Reader{
buf: buf,
}
}
// Read reads bytes into the given byte slice and returns the number of bytes read and an error if occurred.
func (r *Reader) Read(b []byte) (n int, err error) {
if len(b) == 0 {
return 0, nil
}
if r.pos >= len(r.buf) {
return 0, io.EOF
}
n = copy(b, r.buf[r.pos:])
r.pos += n
return
}
// Bytes returns the underlying byte slice.
func (r *Reader) Bytes() []byte {
return r.buf
}
// Reset resets the position of the read pointer to the beginning of the underlying byte slice.
func (r *Reader) Reset() {
r.pos = 0
}
// Len returns the length of the buffer.
func (r *Reader) Len() int {
return len(r.buf)
}
parse-2.5.21/buffer/reader_test.go 0000664 0000000 0000000 00000002331 14117006534 0017014 0 ustar 00root root 0000000 0000000 package buffer
import (
"bytes"
"fmt"
"io"
"testing"
"github.com/tdewolff/test"
)
func TestReader(t *testing.T) {
s := []byte("abcde")
r := NewReader(s)
test.T(t, r.Len(), 5, "len")
test.Bytes(t, r.Bytes(), s, "reader must return bytes stored")
buf := make([]byte, 3)
n, err := r.Read(buf)
test.T(t, err, nil, "error")
test.That(t, n == 3, "first read must read 3 characters")
test.Bytes(t, buf, []byte("abc"), "first read must match 'abc'")
n, err = r.Read(buf)
test.T(t, err, nil, "error")
test.That(t, n == 2, "second read must read 2 characters")
test.Bytes(t, buf[:n], []byte("de"), "second read must match 'de'")
n, err = r.Read(buf)
test.T(t, err, io.EOF, "error")
test.That(t, n == 0, "third read must read 0 characters")
n, err = r.Read(nil)
test.T(t, err, nil, "error")
test.That(t, n == 0, "read to nil buffer must return 0 characters read")
r.Reset()
n, err = r.Read(buf)
test.T(t, err, nil, "error")
test.That(t, n == 3, "read after reset must read 3 characters")
test.Bytes(t, buf, []byte("abc"), "read after reset must match 'abc'")
}
func ExampleNewReader() {
r := NewReader([]byte("Lorem ipsum"))
w := &bytes.Buffer{}
io.Copy(w, r)
fmt.Println(w.String())
// Output: Lorem ipsum
}
parse-2.5.21/buffer/streamlexer.go 0000664 0000000 0000000 00000013334 14117006534 0017053 0 ustar 00root root 0000000 0000000 package buffer
import (
"io"
)
type block struct {
buf []byte
next int // index in pool plus one
active bool
}
type bufferPool struct {
pool []block
head int // index in pool plus one
tail int // index in pool plus one
pos int // byte pos in tail
}
func (z *bufferPool) swap(oldBuf []byte, size int) []byte {
// find new buffer that can be reused
swap := -1
for i := 0; i < len(z.pool); i++ {
if !z.pool[i].active && size <= cap(z.pool[i].buf) {
swap = i
break
}
}
if swap == -1 { // no free buffer found for reuse
if z.tail == 0 && z.pos >= len(oldBuf) && size <= cap(oldBuf) { // but we can reuse the current buffer!
z.pos -= len(oldBuf)
return oldBuf[:0]
}
// allocate new
z.pool = append(z.pool, block{make([]byte, 0, size), 0, true})
swap = len(z.pool) - 1
}
newBuf := z.pool[swap].buf
// put current buffer into pool
z.pool[swap] = block{oldBuf, 0, true}
if z.head != 0 {
z.pool[z.head-1].next = swap + 1
}
z.head = swap + 1
if z.tail == 0 {
z.tail = swap + 1
}
return newBuf[:0]
}
func (z *bufferPool) free(n int) {
z.pos += n
// move the tail over to next buffers
for z.tail != 0 && z.pos >= len(z.pool[z.tail-1].buf) {
z.pos -= len(z.pool[z.tail-1].buf)
newTail := z.pool[z.tail-1].next
z.pool[z.tail-1].active = false // after this, any thread may pick up the inactive buffer, so it can't be used anymore
z.tail = newTail
}
if z.tail == 0 {
z.head = 0
}
}
// StreamLexer is a buffered reader that allows peeking forward and shifting, taking an io.Reader.
// It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.
type StreamLexer struct {
r io.Reader
err error
pool bufferPool
buf []byte
start int // index in buf
pos int // index in buf
prevStart int
free int
}
// NewStreamLexer returns a new StreamLexer for a given io.Reader with a 4kB estimated buffer size.
// If the io.Reader implements Bytes, that buffer is used instead.
func NewStreamLexer(r io.Reader) *StreamLexer {
return NewStreamLexerSize(r, defaultBufSize)
}
// NewStreamLexerSize returns a new StreamLexer for a given io.Reader and estimated required buffer size.
// If the io.Reader implements Bytes, that buffer is used instead.
func NewStreamLexerSize(r io.Reader, size int) *StreamLexer {
// if reader has the bytes in memory already, use that instead
if buffer, ok := r.(interface {
Bytes() []byte
}); ok {
return &StreamLexer{
err: io.EOF,
buf: buffer.Bytes(),
}
}
return &StreamLexer{
r: r,
buf: make([]byte, 0, size),
}
}
func (z *StreamLexer) read(pos int) byte {
if z.err != nil {
return 0
}
// free unused bytes
z.pool.free(z.free)
z.free = 0
// get new buffer
c := cap(z.buf)
p := pos - z.start + 1
if 2*p > c { // if the token is larger than half the buffer, increase buffer size
c = 2*c + p
}
d := len(z.buf) - z.start
buf := z.pool.swap(z.buf[:z.start], c)
copy(buf[:d], z.buf[z.start:]) // copy the left-overs (unfinished token) from the old buffer
// read in new data for the rest of the buffer
var n int
for pos-z.start >= d && z.err == nil {
n, z.err = z.r.Read(buf[d:cap(buf)])
d += n
}
pos -= z.start
z.pos -= z.start
z.start, z.buf = 0, buf[:d]
if pos >= d {
return 0
}
return z.buf[pos]
}
// Err returns the error returned from io.Reader. It may still return valid bytes for a while though.
func (z *StreamLexer) Err() error {
if z.err == io.EOF && z.pos < len(z.buf) {
return nil
}
return z.err
}
// Free frees up bytes of length n from previously shifted tokens.
// Each call to Shift should at one point be followed by a call to Free with a length returned by ShiftLen.
func (z *StreamLexer) Free(n int) {
z.free += n
}
// Peek returns the ith byte relative to the end position and possibly does an allocation.
// Peek returns zero when an error has occurred, Err returns the error.
// TODO: inline function
func (z *StreamLexer) Peek(pos int) byte {
pos += z.pos
if uint(pos) < uint(len(z.buf)) { // uint for BCE
return z.buf[pos]
}
return z.read(pos)
}
// PeekRune returns the rune and rune length of the ith byte relative to the end position.
func (z *StreamLexer) PeekRune(pos int) (rune, int) {
// from unicode/utf8
c := z.Peek(pos)
if c < 0xC0 {
return rune(c), 1
} else if c < 0xE0 {
return rune(c&0x1F)<<6 | rune(z.Peek(pos+1)&0x3F), 2
} else if c < 0xF0 {
return rune(c&0x0F)<<12 | rune(z.Peek(pos+1)&0x3F)<<6 | rune(z.Peek(pos+2)&0x3F), 3
}
return rune(c&0x07)<<18 | rune(z.Peek(pos+1)&0x3F)<<12 | rune(z.Peek(pos+2)&0x3F)<<6 | rune(z.Peek(pos+3)&0x3F), 4
}
// Move advances the position.
func (z *StreamLexer) Move(n int) {
z.pos += n
}
// Pos returns a mark to which can be rewinded.
func (z *StreamLexer) Pos() int {
return z.pos - z.start
}
// Rewind rewinds the position to the given position.
func (z *StreamLexer) Rewind(pos int) {
z.pos = z.start + pos
}
// Lexeme returns the bytes of the current selection.
func (z *StreamLexer) Lexeme() []byte {
return z.buf[z.start:z.pos]
}
// Skip collapses the position to the end of the selection.
func (z *StreamLexer) Skip() {
z.start = z.pos
}
// Shift returns the bytes of the current selection and collapses the position to the end of the selection.
// It also returns the number of bytes we moved since the last call to Shift. This can be used in calls to Free.
func (z *StreamLexer) Shift() []byte {
if z.pos > len(z.buf) { // make sure we peeked at least as much as we shift
z.read(z.pos - 1)
}
b := z.buf[z.start:z.pos]
z.start = z.pos
return b
}
// ShiftLen returns the number of bytes moved since the last call to ShiftLen. This can be used in calls to Free because it takes into account multiple Shifts or Skips.
func (z *StreamLexer) ShiftLen() int {
n := z.start - z.prevStart
z.prevStart = z.start
return n
}
parse-2.5.21/buffer/streamlexer_test.go 0000664 0000000 0000000 00000012511 14117006534 0020106 0 ustar 00root root 0000000 0000000 package buffer
import (
"bytes"
"io"
"testing"
"github.com/tdewolff/test"
)
func TestBufferPool(t *testing.T) {
z := &bufferPool{}
lorem := []byte("Lorem ipsum")
dolor := []byte("dolor sit amet")
consectetur := []byte("consectetur adipiscing elit")
// set lorem as first buffer and get new dolor buffer
b := z.swap(lorem, len(dolor))
test.That(t, len(b) == 0)
test.That(t, cap(b) == len(dolor))
b = append(b, dolor...)
// free first buffer so it will be reused
z.free(len(lorem))
b = z.swap(b, len(lorem))
b = b[:len(lorem)]
test.Bytes(t, b, lorem)
b = z.swap(b, len(consectetur))
b = append(b, consectetur...)
// free in advance to reuse the same buffer
z.free(len(dolor) + len(lorem) + len(consectetur))
test.That(t, z.head == 0)
b = z.swap(b, len(consectetur))
b = b[:len(consectetur)]
test.Bytes(t, b, consectetur)
// free in advance but request larger buffer
z.free(len(consectetur))
b = z.swap(b, len(consectetur)+1)
b = append(b, consectetur...)
b = append(b, '.')
test.That(t, cap(b) == len(consectetur)+1)
}
func TestStreamLexer(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewStreamLexer(bytes.NewBufferString(s))
test.T(t, z.err, io.EOF, "buffer must be fully in memory")
test.T(t, z.Err(), nil, "buffer is at EOF but must not return EOF until we reach that")
test.That(t, z.Pos() == 0, "buffer must start at position 0")
test.That(t, z.Peek(0) == 'L', "first character must be 'L'")
test.That(t, z.Peek(1) == 'o', "second character must be 'o'")
z.Move(1)
test.That(t, z.Peek(0) == 'o', "must be 'o' at position 1")
test.That(t, z.Peek(1) == 'r', "must be 'r' at position 1")
z.Rewind(6)
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 6")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 7")
test.Bytes(t, z.Lexeme(), []byte("Lorem "), "buffered string must now read 'Lorem ' when at position 6")
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.ShiftLen() == len("Lorem "), "shifted length must equal last shift")
test.That(t, z.Pos() == 0, "after shifting position must be 0")
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 0 after shifting")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 1 after shifting")
test.T(t, z.Err(), nil, "error must be nil at this point")
z.Move(len(s) - len("Lorem ") - 1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer")
z.Skip()
test.That(t, z.Pos() == 0, "after skipping position must be 0")
z.Move(1)
test.T(t, z.Err(), io.EOF, "error must be EOF when past the buffer")
z.Move(-1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer, even when it has been past the buffer")
z.Free(0) // has already been tested
}
func TestStreamLexerShift(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 5)
z.Move(len("Lorem "))
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.ShiftLen() == len("Lorem "), "shifted length must equal last shift")
}
func TestStreamLexerSmall(t *testing.T) {
s := `abcdefghijklm`
z := NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 4)
test.That(t, z.Peek(8) == 'i', "first character must be 'i' at position 8")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 4)
test.That(t, z.Peek(12) == 'm', "first character must be 'm' at position 12")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 0)
test.That(t, z.Peek(4) == 'e', "first character must be 'e' at position 4")
z = NewStreamLexerSize(test.NewPlainReader(bytes.NewBufferString(s)), 13)
test.That(t, z.Peek(13) == 0, "must yield error at position 13")
}
func TestStreamLexerSingle(t *testing.T) {
z := NewStreamLexer(test.NewInfiniteReader())
test.That(t, z.Peek(0) == '.')
test.That(t, z.Peek(1) == '.')
test.That(t, z.Peek(3) == '.', "required two successful reads")
}
func TestStreamLexerRunes(t *testing.T) {
z := NewStreamLexer(bytes.NewBufferString("aæ†\U00100000"))
r, n := z.PeekRune(0)
test.That(t, n == 1, "first character must be length 1")
test.That(t, r == 'a', "first character must be rune 'a'")
r, n = z.PeekRune(1)
test.That(t, n == 2, "second character must be length 2")
test.That(t, r == 'æ', "second character must be rune 'æ'")
r, n = z.PeekRune(3)
test.That(t, n == 3, "fourth character must be length 3")
test.That(t, r == '†', "fourth character must be rune '†'")
r, n = z.PeekRune(6)
test.That(t, n == 4, "seventh character must be length 4")
test.That(t, r == '\U00100000', "seventh character must be rune '\U00100000'")
}
func TestStreamLexerBadRune(t *testing.T) {
z := NewStreamLexer(bytes.NewBufferString("\xF0")) // expect four byte rune
r, n := z.PeekRune(0)
test.T(t, n, 4, "length")
test.T(t, r, rune(0), "rune")
}
func TestStreamLexerZeroLen(t *testing.T) {
z := NewStreamLexer(test.NewPlainReader(bytes.NewBufferString("")))
test.That(t, z.Peek(0) == 0, "first character must yield error")
}
func TestStreamLexerEmptyReader(t *testing.T) {
z := NewStreamLexer(test.NewEmptyReader())
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), io.EOF, "error must be EOF")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
parse-2.5.21/buffer/writer.go 0000664 0000000 0000000 00000002012 14117006534 0016023 0 ustar 00root root 0000000 0000000 package buffer
// Writer implements an io.Writer over a byte slice.
type Writer struct {
buf []byte
}
// NewWriter returns a new Writer for a given byte slice.
func NewWriter(buf []byte) *Writer {
return &Writer{
buf: buf,
}
}
// Write writes bytes from the given byte slice and returns the number of bytes written and an error if occurred. When err != nil, n == 0.
func (w *Writer) Write(b []byte) (int, error) {
n := len(b)
end := len(w.buf)
if end+n > cap(w.buf) {
buf := make([]byte, end, 2*cap(w.buf)+n)
copy(buf, w.buf)
w.buf = buf
}
w.buf = w.buf[:end+n]
return copy(w.buf[end:], b), nil
}
// Len returns the length of the underlying byte slice.
func (w *Writer) Len() int {
return len(w.buf)
}
// Bytes returns the underlying byte slice.
func (w *Writer) Bytes() []byte {
return w.buf
}
// Reset empties and reuses the current buffer. Subsequent writes will overwrite the buffer, so any reference to the underlying slice is invalidated after this call.
func (w *Writer) Reset() {
w.buf = w.buf[:0]
}
parse-2.5.21/buffer/writer_test.go 0000664 0000000 0000000 00000002512 14117006534 0017067 0 ustar 00root root 0000000 0000000 package buffer
import (
"fmt"
"testing"
"github.com/tdewolff/test"
)
func TestWriter(t *testing.T) {
w := NewWriter(make([]byte, 0, 3))
test.That(t, w.Len() == 0, "buffer must initially have zero length")
n, _ := w.Write([]byte("abc"))
test.That(t, n == 3, "first write must write 3 characters")
test.Bytes(t, w.Bytes(), []byte("abc"), "first write must match 'abc'")
test.That(t, w.Len() == 3, "buffer must have length 3 after first write")
n, _ = w.Write([]byte("def"))
test.That(t, n == 3, "second write must write 3 characters")
test.Bytes(t, w.Bytes(), []byte("abcdef"), "second write must match 'abcdef'")
w.Reset()
test.Bytes(t, w.Bytes(), []byte(""), "reset must match ''")
n, _ = w.Write([]byte("ghijkl"))
test.That(t, n == 6, "third write must write 6 characters")
test.Bytes(t, w.Bytes(), []byte("ghijkl"), "third write must match 'ghijkl'")
}
func ExampleNewWriter() {
w := NewWriter(make([]byte, 0, 11)) // initial buffer length is 11
w.Write([]byte("Lorem ipsum"))
fmt.Println(string(w.Bytes()))
// Output: Lorem ipsum
}
func ExampleWriter_Reset() {
w := NewWriter(make([]byte, 0, 11)) // initial buffer length is 10
w.Write([]byte("garbage that will be overwritten")) // does reallocation
w.Reset()
w.Write([]byte("Lorem ipsum"))
fmt.Println(string(w.Bytes()))
// Output: Lorem ipsum
}
parse-2.5.21/common.go 0000664 0000000 0000000 00000012414 14117006534 0014535 0 ustar 00root root 0000000 0000000 // Package parse contains a collection of parsers for various formats in its subpackages.
package parse
import (
"bytes"
"encoding/base64"
"errors"
)
var (
dataSchemeBytes = []byte("data:")
base64Bytes = []byte("base64")
textMimeBytes = []byte("text/plain")
)
// ErrBadDataURI is returned by DataURI when the byte slice does not start with 'data:' or is too short.
var ErrBadDataURI = errors.New("not a data URI")
// Number returns the number of bytes that parse as a number of the regex format (+|-)?([0-9]+(\.[0-9]+)?|\.[0-9]+)((e|E)(+|-)?[0-9]+)?.
func Number(b []byte) int {
if len(b) == 0 {
return 0
}
i := 0
if b[i] == '+' || b[i] == '-' {
i++
if i >= len(b) {
return 0
}
}
firstDigit := (b[i] >= '0' && b[i] <= '9')
if firstDigit {
i++
for i < len(b) && b[i] >= '0' && b[i] <= '9' {
i++
}
}
if i < len(b) && b[i] == '.' {
i++
if i < len(b) && b[i] >= '0' && b[i] <= '9' {
i++
for i < len(b) && b[i] >= '0' && b[i] <= '9' {
i++
}
} else if firstDigit {
// . could belong to the next token
i--
return i
} else {
return 0
}
} else if !firstDigit {
return 0
}
iOld := i
if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
i++
if i < len(b) && (b[i] == '+' || b[i] == '-') {
i++
}
if i >= len(b) || b[i] < '0' || b[i] > '9' {
// e could belong to next token
return iOld
}
for i < len(b) && b[i] >= '0' && b[i] <= '9' {
i++
}
}
return i
}
// Dimension parses a byte-slice and returns the length of the number and its unit.
func Dimension(b []byte) (int, int) {
num := Number(b)
if num == 0 || num == len(b) {
return num, 0
} else if b[num] == '%' {
return num, 1
} else if b[num] >= 'a' && b[num] <= 'z' || b[num] >= 'A' && b[num] <= 'Z' {
i := num + 1
for i < len(b) && (b[i] >= 'a' && b[i] <= 'z' || b[i] >= 'A' && b[i] <= 'Z') {
i++
}
return num, i - num
}
return num, 0
}
// Mediatype parses a given mediatype and splits the mimetype from the parameters.
// It works similar to mime.ParseMediaType but is faster.
func Mediatype(b []byte) ([]byte, map[string]string) {
i := 0
for i < len(b) && b[i] == ' ' {
i++
}
b = b[i:]
n := len(b)
mimetype := b
var params map[string]string
for i := 3; i < n; i++ { // mimetype is at least three characters long
if b[i] == ';' || b[i] == ' ' {
mimetype = b[:i]
if b[i] == ' ' {
i++ // space
for i < n && b[i] == ' ' {
i++
}
if n <= i || b[i] != ';' {
break
}
}
params = map[string]string{}
s := string(b)
PARAM:
i++ // semicolon
for i < n && s[i] == ' ' {
i++
}
start := i
for i < n && s[i] != '=' && s[i] != ';' && s[i] != ' ' {
i++
}
key := s[start:i]
for i < n && s[i] == ' ' {
i++
}
if i < n && s[i] == '=' {
i++
for i < n && s[i] == ' ' {
i++
}
start = i
for i < n && s[i] != ';' && s[i] != ' ' {
i++
}
} else {
start = i
}
params[key] = s[start:i]
for i < n && s[i] == ' ' {
i++
}
if i < n && s[i] == ';' {
goto PARAM
}
break
}
}
return mimetype, params
}
// DataURI parses the given data URI and returns the mediatype, data and ok.
func DataURI(dataURI []byte) ([]byte, []byte, error) {
if len(dataURI) > 5 && bytes.Equal(dataURI[:5], dataSchemeBytes) {
dataURI = dataURI[5:]
inBase64 := false
var mediatype []byte
i := 0
for j := 0; j < len(dataURI); j++ {
c := dataURI[j]
if c == '=' || c == ';' || c == ',' {
if c != '=' && bytes.Equal(TrimWhitespace(dataURI[i:j]), base64Bytes) {
if len(mediatype) > 0 {
mediatype = mediatype[:len(mediatype)-1]
}
inBase64 = true
i = j
} else if c != ',' {
mediatype = append(append(mediatype, TrimWhitespace(dataURI[i:j])...), c)
i = j + 1
} else {
mediatype = append(mediatype, TrimWhitespace(dataURI[i:j])...)
}
if c == ',' {
if len(mediatype) == 0 || mediatype[0] == ';' {
mediatype = textMimeBytes
}
data := dataURI[j+1:]
if inBase64 {
decoded := make([]byte, base64.StdEncoding.DecodedLen(len(data)))
n, err := base64.StdEncoding.Decode(decoded, data)
if err != nil {
return nil, nil, err
}
data = decoded[:n]
} else {
data = DecodeURL(data)
}
return mediatype, data, nil
}
}
}
}
return nil, nil, ErrBadDataURI
}
// QuoteEntity parses the given byte slice and returns the quote that got matched (' or ") and its entity length.
// TODO: deprecated
func QuoteEntity(b []byte) (quote byte, n int) {
if len(b) < 5 || b[0] != '&' {
return 0, 0
}
if b[1] == '#' {
if b[2] == 'x' {
i := 3
for i < len(b) && b[i] == '0' {
i++
}
if i+2 < len(b) && b[i] == '2' && b[i+2] == ';' {
if b[i+1] == '2' {
return '"', i + 3 // "
} else if b[i+1] == '7' {
return '\'', i + 3 // '
}
}
} else {
i := 2
for i < len(b) && b[i] == '0' {
i++
}
if i+2 < len(b) && b[i] == '3' && b[i+2] == ';' {
if b[i+1] == '4' {
return '"', i + 3 // "
} else if b[i+1] == '9' {
return '\'', i + 3 // '
}
}
}
} else if len(b) >= 6 && b[5] == ';' {
if bytes.Equal(b[1:5], []byte{'q', 'u', 'o', 't'}) {
return '"', 6 // "
} else if bytes.Equal(b[1:5], []byte{'a', 'p', 'o', 's'}) {
return '\'', 6 // '
}
}
return 0, 0
}
parse-2.5.21/common_test.go 0000664 0000000 0000000 00000010740 14117006534 0015574 0 ustar 00root root 0000000 0000000 package parse
import (
"encoding/base64"
"mime"
"testing"
"github.com/tdewolff/test"
)
func TestParseNumber(t *testing.T) {
var numberTests = []struct {
number string
expected int
}{
{"5", 1},
{"0.51", 4},
{"0.5e-99", 7},
{"0.5e-", 3},
{"+50.0", 5},
{".0", 2},
{"0.", 1},
{"", 0},
{"+", 0},
{".", 0},
{"a", 0},
}
for _, tt := range numberTests {
t.Run(tt.number, func(t *testing.T) {
n := Number([]byte(tt.number))
test.T(t, n, tt.expected)
})
}
}
func TestParseDimension(t *testing.T) {
var dimensionTests = []struct {
dimension string
expectedNum int
expectedUnit int
}{
{"5px", 1, 2},
{"5px ", 1, 2},
{"5%", 1, 1},
{"5em", 1, 2},
{"px", 0, 0},
{"1", 1, 0},
{"1~", 1, 0},
}
for _, tt := range dimensionTests {
t.Run(tt.dimension, func(t *testing.T) {
num, unit := Dimension([]byte(tt.dimension))
test.T(t, num, tt.expectedNum, "number")
test.T(t, unit, tt.expectedUnit, "unit")
})
}
}
func TestMediatype(t *testing.T) {
var mediatypeTests = []struct {
mediatype string
expectedMimetype string
expectedParams map[string]string
}{
{"text/plain", "text/plain", nil},
{"text/plain;charset=US-ASCII", "text/plain", map[string]string{"charset": "US-ASCII"}},
{" text/plain ; charset = US-ASCII ", "text/plain", map[string]string{"charset": "US-ASCII"}},
{" text/plain a", "text/plain", nil},
{"text/plain;base64", "text/plain", map[string]string{"base64": ""}},
{"text/plain;inline=;base64", "text/plain", map[string]string{"inline": "", "base64": ""}},
{"ÿ ", "ÿ ", nil}, // OSS-Fuzz; ÿ is two bytes in UTF8
{"ÿ ;", "ÿ ", map[string]string{"": ""}},
}
for _, tt := range mediatypeTests {
t.Run(tt.mediatype, func(t *testing.T) {
mimetype, params := Mediatype([]byte(tt.mediatype))
test.String(t, string(mimetype), tt.expectedMimetype, "mimetype")
test.T(t, params, tt.expectedParams, "parameters") // TODO
})
}
}
func TestParseDataURI(t *testing.T) {
var dataURITests = []struct {
dataURI string
expectedMimetype string
expectedData string
expectedErr error
}{
{"www.domain.com", "", "", ErrBadDataURI},
{"data:,", "text/plain", "", nil},
{"data:text/xml,", "text/xml", "", nil},
{"data:,text", "text/plain", "text", nil},
{"data:;base64,dGV4dA==", "text/plain", "text", nil},
{"data:image/svg+xml,", "image/svg+xml", "", nil},
{"data:;base64,()", "", "", base64.CorruptInputError(0)},
{"data:image/svg+xml,%3Cpath%20stroke-width='9.38%'/%3E", "image/svg+xml", "", nil},
{"data:,%ii", "text/plain", "%ii", nil},
}
for _, tt := range dataURITests {
t.Run(tt.dataURI, func(t *testing.T) {
mimetype, data, err := DataURI([]byte(tt.dataURI))
test.T(t, err, tt.expectedErr)
test.String(t, string(mimetype), tt.expectedMimetype, "mimetype")
test.String(t, string(data), tt.expectedData, "data")
})
}
}
func TestParseQuoteEntity(t *testing.T) {
var quoteEntityTests = []struct {
quoteEntity string
expectedQuote byte
expectedN int
}{
{""", '"', 5},
{"'", '\'', 6},
{""", '"', 8},
{"'", '\'', 6},
{""", '"', 6},
{"'", '\'', 6},
{">", 0x00, 0},
{"&", 0x00, 0},
}
for _, tt := range quoteEntityTests {
t.Run(tt.quoteEntity, func(t *testing.T) {
quote, n := QuoteEntity([]byte(tt.quoteEntity))
test.T(t, quote, tt.expectedQuote, "quote")
test.T(t, n, tt.expectedN, "quote length")
})
}
}
////////////////////////////////////////////////////////////////
func BenchmarkParseMediatypeStd(b *testing.B) {
mediatype := "text/plain"
for i := 0; i < b.N; i++ {
mime.ParseMediaType(mediatype)
}
}
func BenchmarkParseMediatypeParamStd(b *testing.B) {
mediatype := "text/plain;inline=1"
for i := 0; i < b.N; i++ {
mime.ParseMediaType(mediatype)
}
}
func BenchmarkParseMediatypeParamsStd(b *testing.B) {
mediatype := "text/plain;charset=US-ASCII;language=US-EN;compression=gzip;base64"
for i := 0; i < b.N; i++ {
mime.ParseMediaType(mediatype)
}
}
func BenchmarkParseMediatypeParse(b *testing.B) {
mediatype := []byte("text/plain")
for i := 0; i < b.N; i++ {
Mediatype(mediatype)
}
}
func BenchmarkParseMediatypeParamParse(b *testing.B) {
mediatype := []byte("text/plain;inline=1")
for i := 0; i < b.N; i++ {
Mediatype(mediatype)
}
}
func BenchmarkParseMediatypeParamsParse(b *testing.B) {
mediatype := []byte("text/plain;charset=US-ASCII;language=US-EN;compression=gzip;base64")
for i := 0; i < b.N; i++ {
Mediatype(mediatype)
}
}
parse-2.5.21/css/ 0000775 0000000 0000000 00000000000 14117006534 0013504 5 ustar 00root root 0000000 0000000 parse-2.5.21/css/README.md 0000664 0000000 0000000 00000007406 14117006534 0014772 0 ustar 00root root 0000000 0000000 # CSS [](https://pkg.go.dev/github.com/tdewolff/parse/v2/css?tab=doc)
This package is a CSS3 lexer and parser written in [Go][1]. Both follow the specification at [CSS Syntax Module Level 3](http://www.w3.org/TR/css-syntax-3/). The lexer takes an io.Reader and converts it into tokens until the EOF. The parser returns a parse tree of the full io.Reader input stream, but the low-level `Next` function can be used for stream parsing to returns grammar units until the EOF.
## Installation
Run the following command
go get -u github.com/tdewolff/parse/v2/css
or add the following import and run project with `go get`
import "github.com/tdewolff/parse/v2/css"
## Lexer
### Usage
The following initializes a new Lexer with io.Reader `r`:
``` go
l := css.NewLexer(parse.NewInput(r))
```
To tokenize until EOF an error, use:
``` go
for {
tt, text := l.Next()
switch tt {
case css.ErrorToken:
// error or EOF set in l.Err()
return
// ...
}
}
```
All tokens (see [CSS Syntax Module Level 3](http://www.w3.org/TR/css3-syntax/)):
``` go
ErrorToken // non-official token, returned when errors occur
IdentToken
FunctionToken // rgb( rgba( ...
AtKeywordToken // @abc
HashToken // #abc
StringToken
BadStringToken
URLToken // url(
BadURLToken
DelimToken // any unmatched character
NumberToken // 5
PercentageToken // 5%
DimensionToken // 5em
UnicodeRangeToken
IncludeMatchToken // ~=
DashMatchToken // |=
PrefixMatchToken // ^=
SuffixMatchToken // $=
SubstringMatchToken // *=
ColumnToken // ||
WhitespaceToken
CDOToken //
ColonToken
SemicolonToken
CommaToken
BracketToken // ( ) [ ] { }, all bracket tokens use this, Data() can distinguish between the brackets
CommentToken // non-official token
```
### Examples
``` go
package main
import (
"os"
"github.com/tdewolff/parse/v2/css"
)
// Tokenize CSS3 from stdin.
func main() {
l := css.NewLexer(parse.NewInput(os.Stdin))
for {
tt, text := l.Next()
switch tt {
case css.ErrorToken:
if l.Err() != io.EOF {
fmt.Println("Error on line", l.Line(), ":", l.Err())
}
return
case css.IdentToken:
fmt.Println("Identifier", string(text))
case css.NumberToken:
fmt.Println("Number", string(text))
// ...
}
}
}
```
## Parser
### Usage
The following creates a new Parser.
``` go
// true because this is the content of an inline style attribute
p := css.NewParser(parse.NewInput(bytes.NewBufferString("color: red;")), true)
```
To iterate over the stylesheet, use:
``` go
for {
gt, _, data := p.Next()
if gt == css.ErrorGrammar {
break
}
// ...
}
```
All grammar units returned by `Next`:
``` go
ErrorGrammar
AtRuleGrammar
EndAtRuleGrammar
RulesetGrammar
EndRulesetGrammar
DeclarationGrammar
TokenGrammar
```
### Examples
``` go
package main
import (
"bytes"
"fmt"
"github.com/tdewolff/parse/v2/css"
)
func main() {
// true because this is the content of an inline style attribute
p := css.NewParser(parse.NewInput(bytes.NewBufferString("color: red;")), true)
out := ""
for {
gt, _, data := p.Next()
if gt == css.ErrorGrammar {
break
} else if gt == css.AtRuleGrammar || gt == css.BeginAtRuleGrammar || gt == css.BeginRulesetGrammar || gt == css.DeclarationGrammar {
out += string(data)
if gt == css.DeclarationGrammar {
out += ":"
}
for _, val := range p.Values() {
out += string(val.Data)
}
if gt == css.BeginAtRuleGrammar || gt == css.BeginRulesetGrammar {
out += "{"
} else if gt == css.AtRuleGrammar || gt == css.DeclarationGrammar {
out += ";"
}
} else {
out += string(data)
}
}
fmt.Println(out)
}
```
## License
Released under the [MIT license](https://github.com/tdewolff/parse/blob/master/LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/css/hash.go 0000664 0000000 0000000 00000003445 14117006534 0014764 0 ustar 00root root 0000000 0000000 package css
// generated by hasher -type=Hash -file=hash.go; DO NOT EDIT, except for adding more constants to the list and rerun go generate
// uses github.com/tdewolff/hasher
//go:generate hasher -type=Hash -file=hash.go
// Hash defines perfect hashes for a predefined list of strings
type Hash uint32
// Unique hash definitions to be used instead of strings
const (
Document Hash = 0x8 // document
Font_Face Hash = 0x809 // font-face
Keyframes Hash = 0x1109 // keyframes
Media Hash = 0x2105 // media
Page Hash = 0x2604 // page
Supports Hash = 0x1908 // supports
)
// String returns the hash' name.
func (i Hash) String() string {
start := uint32(i >> 8)
n := uint32(i & 0xff)
if start+n > uint32(len(_Hash_text)) {
return ""
}
return _Hash_text[start : start+n]
}
// ToHash returns the hash whose name is s. It returns zero if there is no
// such hash. It is case sensitive.
func ToHash(s []byte) Hash {
if len(s) == 0 || len(s) > _Hash_maxLen {
return 0
}
h := uint32(_Hash_hash0)
for i := 0; i < len(s); i++ {
h ^= uint32(s[i])
h *= 16777619
}
if i := _Hash_table[h&uint32(len(_Hash_table)-1)]; int(i&0xff) == len(s) {
t := _Hash_text[i>>8 : i>>8+i&0xff]
for i := 0; i < len(s); i++ {
if t[i] != s[i] {
goto NEXT
}
}
return i
}
NEXT:
if i := _Hash_table[(h>>16)&uint32(len(_Hash_table)-1)]; int(i&0xff) == len(s) {
t := _Hash_text[i>>8 : i>>8+i&0xff]
for i := 0; i < len(s); i++ {
if t[i] != s[i] {
return 0
}
}
return i
}
return 0
}
const _Hash_hash0 = 0x9acb0442
const _Hash_maxLen = 9
const _Hash_text = "documentfont-facekeyframesupportsmediapage"
var _Hash_table = [1 << 3]Hash{
0x1: 0x2604, // page
0x2: 0x2105, // media
0x3: 0x809, // font-face
0x5: 0x1109, // keyframes
0x6: 0x1908, // supports
0x7: 0x8, // document
}
parse-2.5.21/css/lex.go 0000664 0000000 0000000 00000034775 14117006534 0014643 0 ustar 00root root 0000000 0000000 // Package css is a CSS3 lexer and parser following the specifications at http://www.w3.org/TR/css-syntax-3/.
package css
// TODO: \uFFFD replacement character for NULL bytes in strings for example, or atleast don't end the string early
import (
"bytes"
"io"
"strconv"
"github.com/tdewolff/parse/v2"
)
// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint32
// TokenType values.
const (
ErrorToken TokenType = iota // extra token when errors occur
IdentToken
FunctionToken // rgb( rgba( ...
AtKeywordToken // @abc
HashToken // #abc
StringToken
BadStringToken
URLToken
BadURLToken
DelimToken // any unmatched character
NumberToken // 5
PercentageToken // 5%
DimensionToken // 5em
UnicodeRangeToken // U+554A
IncludeMatchToken // ~=
DashMatchToken // |=
PrefixMatchToken // ^=
SuffixMatchToken // $=
SubstringMatchToken // *=
ColumnToken // ||
WhitespaceToken // space \t \r \n \f
CDOToken //
ColonToken // :
SemicolonToken // ;
CommaToken // ,
LeftBracketToken // [
RightBracketToken // ]
LeftParenthesisToken // (
RightParenthesisToken // )
LeftBraceToken // {
RightBraceToken // }
CommentToken // extra token for comments
EmptyToken
CustomPropertyNameToken
CustomPropertyValueToken
)
// String returns the string representation of a TokenType.
func (tt TokenType) String() string {
switch tt {
case ErrorToken:
return "Error"
case IdentToken:
return "Ident"
case FunctionToken:
return "Function"
case AtKeywordToken:
return "AtKeyword"
case HashToken:
return "Hash"
case StringToken:
return "String"
case BadStringToken:
return "BadString"
case URLToken:
return "URL"
case BadURLToken:
return "BadURL"
case DelimToken:
return "Delim"
case NumberToken:
return "Number"
case PercentageToken:
return "Percentage"
case DimensionToken:
return "Dimension"
case UnicodeRangeToken:
return "UnicodeRange"
case IncludeMatchToken:
return "IncludeMatch"
case DashMatchToken:
return "DashMatch"
case PrefixMatchToken:
return "PrefixMatch"
case SuffixMatchToken:
return "SuffixMatch"
case SubstringMatchToken:
return "SubstringMatch"
case ColumnToken:
return "Column"
case WhitespaceToken:
return "Whitespace"
case CDOToken:
return "CDO"
case CDCToken:
return "CDC"
case ColonToken:
return "Colon"
case SemicolonToken:
return "Semicolon"
case CommaToken:
return "Comma"
case LeftBracketToken:
return "LeftBracket"
case RightBracketToken:
return "RightBracket"
case LeftParenthesisToken:
return "LeftParenthesis"
case RightParenthesisToken:
return "RightParenthesis"
case LeftBraceToken:
return "LeftBrace"
case RightBraceToken:
return "RightBrace"
case CommentToken:
return "Comment"
case EmptyToken:
return "Empty"
case CustomPropertyNameToken:
return "CustomPropertyName"
case CustomPropertyValueToken:
return "CustomPropertyValue"
}
return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}
////////////////////////////////////////////////////////////////
// Lexer is the state for the lexer.
type Lexer struct {
r *parse.Input
}
// NewLexer returns a new Lexer for a given io.Reader.
func NewLexer(r *parse.Input) *Lexer {
return &Lexer{
r: r,
}
}
// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
func (l *Lexer) Err() error {
return l.r.Err()
}
// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (l *Lexer) Next() (TokenType, []byte) {
switch l.r.Peek(0) {
case ' ', '\t', '\n', '\r', '\f':
l.r.Move(1)
for l.consumeWhitespace() {
}
return WhitespaceToken, l.r.Shift()
case ':':
l.r.Move(1)
return ColonToken, l.r.Shift()
case ';':
l.r.Move(1)
return SemicolonToken, l.r.Shift()
case ',':
l.r.Move(1)
return CommaToken, l.r.Shift()
case '(', ')', '[', ']', '{', '}':
if t := l.consumeBracket(); t != ErrorToken {
return t, l.r.Shift()
}
case '#':
if l.consumeHashToken() {
return HashToken, l.r.Shift()
}
case '"', '\'':
if t := l.consumeString(); t != ErrorToken {
return t, l.r.Shift()
}
case '.', '+':
if t := l.consumeNumeric(); t != ErrorToken {
return t, l.r.Shift()
}
case '-':
if t := l.consumeNumeric(); t != ErrorToken {
return t, l.r.Shift()
} else if t := l.consumeIdentlike(); t != ErrorToken {
return t, l.r.Shift()
} else if l.consumeCDCToken() {
return CDCToken, l.r.Shift()
} else if l.consumeCustomVariableToken() {
return CustomPropertyNameToken, l.r.Shift()
}
case '@':
if l.consumeAtKeywordToken() {
return AtKeywordToken, l.r.Shift()
}
case '$', '*', '^', '~':
if t := l.consumeMatch(); t != ErrorToken {
return t, l.r.Shift()
}
case '/':
if l.consumeComment() {
return CommentToken, l.r.Shift()
}
case '<':
if l.consumeCDOToken() {
return CDOToken, l.r.Shift()
}
case '\\':
if t := l.consumeIdentlike(); t != ErrorToken {
return t, l.r.Shift()
}
case 'u', 'U':
if l.consumeUnicodeRangeToken() {
return UnicodeRangeToken, l.r.Shift()
} else if t := l.consumeIdentlike(); t != ErrorToken {
return t, l.r.Shift()
}
case '|':
if t := l.consumeMatch(); t != ErrorToken {
return t, l.r.Shift()
} else if l.consumeColumnToken() {
return ColumnToken, l.r.Shift()
}
case 0:
if l.r.Err() != nil {
return ErrorToken, nil
}
default:
if t := l.consumeNumeric(); t != ErrorToken {
return t, l.r.Shift()
} else if t := l.consumeIdentlike(); t != ErrorToken {
return t, l.r.Shift()
}
}
// can't be rune because consumeIdentlike consumes that as an identifier
l.r.Move(1)
return DelimToken, l.r.Shift()
}
////////////////////////////////////////////////////////////////
/*
The following functions follow the railroad diagrams in http://www.w3.org/TR/css3-syntax/
*/
func (l *Lexer) consumeByte(c byte) bool {
if l.r.Peek(0) == c {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeComment() bool {
if l.r.Peek(0) != '/' || l.r.Peek(1) != '*' {
return false
}
l.r.Move(2)
for {
c := l.r.Peek(0)
if c == 0 && l.r.Err() != nil {
break
} else if c == '*' && l.r.Peek(1) == '/' {
l.r.Move(2)
return true
}
l.r.Move(1)
}
return true
}
func (l *Lexer) consumeNewline() bool {
c := l.r.Peek(0)
if c == '\n' || c == '\f' {
l.r.Move(1)
return true
} else if c == '\r' {
if l.r.Peek(1) == '\n' {
l.r.Move(2)
} else {
l.r.Move(1)
}
return true
}
return false
}
func (l *Lexer) consumeWhitespace() bool {
c := l.r.Peek(0)
if c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeDigit() bool {
c := l.r.Peek(0)
if c >= '0' && c <= '9' {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeHexDigit() bool {
c := l.r.Peek(0)
if (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeEscape() bool {
if l.r.Peek(0) != '\\' {
return false
}
mark := l.r.Pos()
l.r.Move(1)
if l.consumeNewline() {
l.r.Rewind(mark)
return false
} else if l.consumeHexDigit() {
for k := 1; k < 6; k++ {
if !l.consumeHexDigit() {
break
}
}
l.consumeWhitespace()
return true
} else {
c := l.r.Peek(0)
if c >= 0xC0 {
_, n := l.r.PeekRune(0)
l.r.Move(n)
return true
} else if c == 0 && l.r.Err() != nil {
l.r.Rewind(mark)
return false
}
}
l.r.Move(1)
return true
}
func (l *Lexer) consumeIdentToken() bool {
mark := l.r.Pos()
if l.r.Peek(0) == '-' {
l.r.Move(1)
}
c := l.r.Peek(0)
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || c == '_' || c >= 0x80) {
if c != '\\' || !l.consumeEscape() {
l.r.Rewind(mark)
return false
}
} else {
l.r.Move(1)
}
for {
c := l.r.Peek(0)
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c >= 0x80) {
if c != '\\' || !l.consumeEscape() {
break
}
} else {
l.r.Move(1)
}
}
return true
}
// support custom variables, https://www.w3.org/TR/css-variables-1/
func (l *Lexer) consumeCustomVariableToken() bool {
// expect to be on a '-'
l.r.Move(1)
if l.r.Peek(0) != '-' {
l.r.Move(-1)
return false
}
if !l.consumeIdentToken() {
l.r.Move(-1)
return false
}
return true
}
func (l *Lexer) consumeAtKeywordToken() bool {
// expect to be on an '@'
l.r.Move(1)
if !l.consumeIdentToken() {
l.r.Move(-1)
return false
}
return true
}
func (l *Lexer) consumeHashToken() bool {
// expect to be on a '#'
mark := l.r.Pos()
l.r.Move(1)
c := l.r.Peek(0)
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c >= 0x80) {
if c != '\\' || !l.consumeEscape() {
l.r.Rewind(mark)
return false
}
} else {
l.r.Move(1)
}
for {
c := l.r.Peek(0)
if !((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || c == '_' || c == '-' || c >= 0x80) {
if c != '\\' || !l.consumeEscape() {
break
}
} else {
l.r.Move(1)
}
}
return true
}
func (l *Lexer) consumeNumberToken() bool {
mark := l.r.Pos()
c := l.r.Peek(0)
if c == '+' || c == '-' {
l.r.Move(1)
}
firstDigit := l.consumeDigit()
if firstDigit {
for l.consumeDigit() {
}
}
if l.r.Peek(0) == '.' {
l.r.Move(1)
if l.consumeDigit() {
for l.consumeDigit() {
}
} else if firstDigit {
// . could belong to the next token
l.r.Move(-1)
return true
} else {
l.r.Rewind(mark)
return false
}
} else if !firstDigit {
l.r.Rewind(mark)
return false
}
mark = l.r.Pos()
c = l.r.Peek(0)
if c == 'e' || c == 'E' {
l.r.Move(1)
c = l.r.Peek(0)
if c == '+' || c == '-' {
l.r.Move(1)
}
if !l.consumeDigit() {
// e could belong to next token
l.r.Rewind(mark)
return true
}
for l.consumeDigit() {
}
}
return true
}
func (l *Lexer) consumeUnicodeRangeToken() bool {
c := l.r.Peek(0)
if (c != 'u' && c != 'U') || l.r.Peek(1) != '+' {
return false
}
mark := l.r.Pos()
l.r.Move(2)
// consume up to 6 hexDigits
k := 0
for l.consumeHexDigit() {
k++
}
// either a minus or a question mark or the end is expected
if l.consumeByte('-') {
if k == 0 || 6 < k {
l.r.Rewind(mark)
return false
}
// consume another up to 6 hexDigits
if l.consumeHexDigit() {
k = 1
for l.consumeHexDigit() {
k++
}
} else {
l.r.Rewind(mark)
return false
}
} else if l.consumeByte('?') {
// could be filled up to 6 characters with question marks or else regular hexDigits
k++
for l.consumeByte('?') {
k++
}
}
if k == 0 || 6 < k {
l.r.Rewind(mark)
return false
}
return true
}
func (l *Lexer) consumeColumnToken() bool {
if l.r.Peek(0) == '|' && l.r.Peek(1) == '|' {
l.r.Move(2)
return true
}
return false
}
func (l *Lexer) consumeCDOToken() bool {
if l.r.Peek(0) == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
l.r.Move(4)
return true
}
return false
}
func (l *Lexer) consumeCDCToken() bool {
if l.r.Peek(0) == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
l.r.Move(3)
return true
}
return false
}
////////////////////////////////////////////////////////////////
// consumeMatch consumes any MatchToken.
func (l *Lexer) consumeMatch() TokenType {
if l.r.Peek(1) == '=' {
switch l.r.Peek(0) {
case '~':
l.r.Move(2)
return IncludeMatchToken
case '|':
l.r.Move(2)
return DashMatchToken
case '^':
l.r.Move(2)
return PrefixMatchToken
case '$':
l.r.Move(2)
return SuffixMatchToken
case '*':
l.r.Move(2)
return SubstringMatchToken
}
}
return ErrorToken
}
// consumeBracket consumes any bracket token.
func (l *Lexer) consumeBracket() TokenType {
switch l.r.Peek(0) {
case '(':
l.r.Move(1)
return LeftParenthesisToken
case ')':
l.r.Move(1)
return RightParenthesisToken
case '[':
l.r.Move(1)
return LeftBracketToken
case ']':
l.r.Move(1)
return RightBracketToken
case '{':
l.r.Move(1)
return LeftBraceToken
case '}':
l.r.Move(1)
return RightBraceToken
}
return ErrorToken
}
// consumeNumeric consumes NumberToken, PercentageToken or DimensionToken.
func (l *Lexer) consumeNumeric() TokenType {
if l.consumeNumberToken() {
if l.consumeByte('%') {
return PercentageToken
} else if l.consumeIdentToken() {
return DimensionToken
}
return NumberToken
}
return ErrorToken
}
// consumeString consumes a string and may return BadStringToken when a newline is encountered.
func (l *Lexer) consumeString() TokenType {
// assume to be on " or '
delim := l.r.Peek(0)
l.r.Move(1)
for {
c := l.r.Peek(0)
if c == 0 && l.r.Err() != nil {
break
} else if c == '\n' || c == '\r' || c == '\f' {
l.r.Move(1)
return BadStringToken
} else if c == delim {
l.r.Move(1)
break
} else if c == '\\' {
if !l.consumeEscape() {
// either newline or EOF after backslash
l.r.Move(1)
l.consumeNewline()
}
} else {
l.r.Move(1)
}
}
return StringToken
}
func (l *Lexer) consumeUnquotedURL() bool {
for {
c := l.r.Peek(0)
if c == 0 && l.r.Err() != nil || c == ')' {
break
} else if c == '"' || c == '\'' || c == '(' || c == '\\' || c == ' ' || c <= 0x1F || c == 0x7F {
if c != '\\' || !l.consumeEscape() {
return false
}
} else {
l.r.Move(1)
}
}
return true
}
// consumeRemnantsBadUrl consumes bytes of a BadUrlToken so that normal tokenization may continue.
func (l *Lexer) consumeRemnantsBadURL() {
for {
if l.consumeByte(')') || l.r.Err() != nil {
break
} else if !l.consumeEscape() {
l.r.Move(1)
}
}
}
// consumeIdentlike consumes IdentToken, FunctionToken or UrlToken.
func (l *Lexer) consumeIdentlike() TokenType {
if l.consumeIdentToken() {
if l.r.Peek(0) != '(' {
return IdentToken
} else if !parse.EqualFold(bytes.Replace(l.r.Lexeme(), []byte{'\\'}, nil, -1), []byte{'u', 'r', 'l'}) {
l.r.Move(1)
return FunctionToken
}
l.r.Move(1)
// consume url
for l.consumeWhitespace() {
}
if c := l.r.Peek(0); c == '"' || c == '\'' {
if l.consumeString() == BadStringToken {
l.consumeRemnantsBadURL()
return BadURLToken
}
} else if !l.consumeUnquotedURL() && !l.consumeWhitespace() { // if unquoted URL fails due to encountering whitespace, continue
l.consumeRemnantsBadURL()
return BadURLToken
}
for l.consumeWhitespace() {
}
if !l.consumeByte(')') && l.r.Err() != io.EOF {
l.consumeRemnantsBadURL()
return BadURLToken
}
return URLToken
}
return ErrorToken
}
parse-2.5.21/css/lex_test.go 0000664 0000000 0000000 00000020574 14117006534 0015672 0 ustar 00root root 0000000 0000000 package css
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type TTs []TokenType
func TestTokens(t *testing.T) {
var tokenTests = []struct {
css string
ttypes []TokenType
lexemes []string
}{
{" ", TTs{}, []string{}},
{"5.2 .4", TTs{NumberToken, NumberToken}, []string{"5.2", ".4"}},
{"color: red;", TTs{IdentToken, ColonToken, IdentToken, SemicolonToken}, []string{"color", ":", "red", ";"}},
{"background: url(\"http://x\");", TTs{IdentToken, ColonToken, URLToken, SemicolonToken}, []string{"background", ":", `url("http://x")`, ";"}},
{"background: URL(x.png);", TTs{IdentToken, ColonToken, URLToken, SemicolonToken}, []string{"background", ":", "URL(x.png)", ";"}},
{"color: rgb(4, 0%, 5em);", TTs{IdentToken, ColonToken, FunctionToken, NumberToken, CommaToken, PercentageToken, CommaToken, DimensionToken, RightParenthesisToken, SemicolonToken}, []string{"color", ":", "rgb(", "4", ",", "0%", ",", "5em", ")", ";"}},
{"body { \"string\" }", TTs{IdentToken, LeftBraceToken, StringToken, RightBraceToken}, []string{"body", "{", `"string"`, "}"}},
{"body { \"str\\\"ing\" }", TTs{IdentToken, LeftBraceToken, StringToken, RightBraceToken}, []string{"body", "{", `"str\"ing"`, "}"}},
{".class { }", TTs{DelimToken, IdentToken, LeftBraceToken, RightBraceToken}, []string{".", "class", "{", "}"}},
{"#class { }", TTs{HashToken, LeftBraceToken, RightBraceToken}, []string{"#class", "{", "}"}},
{"#class\\#withhash { }", TTs{HashToken, LeftBraceToken, RightBraceToken}, []string{`#class\#withhash`, "{", "}"}},
{"@media print { }", TTs{AtKeywordToken, IdentToken, LeftBraceToken, RightBraceToken}, []string{"@media", "print", "{", "}"}},
{"/*comment*/", TTs{CommentToken}, []string{"/*comment*/"}},
{"/*com* /ment*/", TTs{CommentToken}, []string{"/*com* /ment*/"}},
{"~= |= ^= $= *=", TTs{IncludeMatchToken, DashMatchToken, PrefixMatchToken, SuffixMatchToken, SubstringMatchToken}, []string{"~=", "|=", "^=", "$=", "*="}},
{"||", TTs{ColumnToken}, []string{"||"}},
{"", TTs{CDOToken, CDCToken}, []string{""}},
{"U+1234", TTs{UnicodeRangeToken}, []string{"U+1234"}},
{"5.2 .4 4e-22", TTs{NumberToken, NumberToken, NumberToken}, []string{"5.2", ".4", "4e-22"}},
{"--custom-variable", TTs{CustomPropertyNameToken}, []string{"--custom-variable"}},
// unexpected ending
{"ident", TTs{IdentToken}, []string{"ident"}},
{"123.", TTs{NumberToken, DelimToken}, []string{"123", "."}},
{"\"string", TTs{StringToken}, []string{`"string`}},
{"123/*comment", TTs{NumberToken, CommentToken}, []string{"123", "/*comment"}},
{"U+1-", TTs{IdentToken, NumberToken, DelimToken}, []string{"U", "+1", "-"}},
// unicode
{"fooδbar", TTs{IdentToken}, []string{"fooδbar"}},
{"foo\\æ\\†", TTs{IdentToken}, []string{"foo\\æ\\†"}},
{"'foo\u554abar'", TTs{StringToken}, []string{"'foo\u554abar'"}},
{"\\000026B", TTs{IdentToken}, []string{"\\000026B"}},
{"\\26 B", TTs{IdentToken}, []string{"\\26 B"}},
// hacks
{`\-\mo\z\-b\i\nd\in\g:\url(//business\i\nfo.co.uk\/labs\/xbl\/xbl\.xml\#xss);`, TTs{IdentToken, ColonToken, URLToken, SemicolonToken}, []string{`\-\mo\z\-b\i\nd\in\g`, ":", `\url(//business\i\nfo.co.uk\/labs\/xbl\/xbl\.xml\#xss)`, ";"}},
{"width/**/:/**/ 40em;", TTs{IdentToken, CommentToken, ColonToken, CommentToken, DimensionToken, SemicolonToken}, []string{"width", "/**/", ":", "/**/", "40em", ";"}},
{":root *> #quince", TTs{ColonToken, IdentToken, DelimToken, DelimToken, HashToken}, []string{":", "root", "*", ">", "#quince"}},
{"html[xmlns*=\"\"]:root", TTs{IdentToken, LeftBracketToken, IdentToken, SubstringMatchToken, StringToken, RightBracketToken, ColonToken, IdentToken}, []string{"html", "[", "xmlns", "*=", `""`, "]", ":", "root"}},
{"body:nth-of-type(1)", TTs{IdentToken, ColonToken, FunctionToken, NumberToken, RightParenthesisToken}, []string{"body", ":", "nth-of-type(", "1", ")"}},
{"color/*\\**/: blue\\9;", TTs{IdentToken, CommentToken, ColonToken, IdentToken, SemicolonToken}, []string{"color", `/*\**/`, ":", `blue\9`, ";"}},
{"color: blue !ie;", TTs{IdentToken, ColonToken, IdentToken, DelimToken, IdentToken, SemicolonToken}, []string{"color", ":", "blue", "!", "ie", ";"}},
// escapes, null and replacement character
{"c\\\x00olor: white;", TTs{IdentToken, ColonToken, IdentToken, SemicolonToken}, []string{"c\\\x00olor", ":", "white", ";"}},
{"null\\0", TTs{IdentToken}, []string{`null\0`}},
{"\\", TTs{DelimToken}, []string{"\\"}},
{"abc\\", TTs{IdentToken, DelimToken}, []string{"abc", "\\"}},
{"#\\", TTs{DelimToken, DelimToken}, []string{"#", "\\"}},
{"#abc\\", TTs{HashToken, DelimToken}, []string{"#abc", "\\"}},
{"\"abc\\", TTs{StringToken}, []string{"\"abc\\"}}, // should officially not include backslash, but no biggie
{"url(abc\\", TTs{BadURLToken}, []string{"url(abc\\"}},
{"\"a\x00b\"", TTs{StringToken}, []string{"\"a\x00b\""}},
{"a\\\x00b", TTs{IdentToken}, []string{"a\\\x00b"}},
{"url(a\x00b)", TTs{BadURLToken}, []string{"url(a\x00b)"}}, // null character cannot be unquoted
{"/*a\x00b*/", TTs{CommentToken}, []string{"/*a\x00b*/"}},
// coverage
{" \n\r\n\r\"\\\r\n\\\r\"", TTs{StringToken}, []string{"\"\\\r\n\\\r\""}},
{"U+?????? U+ABCD?? U+ABC-DEF", TTs{UnicodeRangeToken, UnicodeRangeToken, UnicodeRangeToken}, []string{"U+??????", "U+ABCD??", "U+ABC-DEF"}},
{"U+? U+A?", TTs{UnicodeRangeToken, UnicodeRangeToken}, []string{"U+?", "U+A?"}},
{"U+ U+ABCDEF?", TTs{IdentToken, DelimToken, IdentToken, DelimToken, IdentToken, DelimToken}, []string{"U", "+", "U", "+", "ABCDEF", "?"}},
{"-5.23 -moz", TTs{NumberToken, IdentToken}, []string{"-5.23", "-moz"}},
{"()", TTs{LeftParenthesisToken, RightParenthesisToken}, []string{"(", ")"}},
{"url( //url\n )", TTs{URLToken}, []string{"url( //url\n )"}},
{"url( ", TTs{URLToken}, []string{"url( "}},
{"url( //url ", TTs{URLToken}, []string{"url( //url "}},
{"url(\")a", TTs{URLToken}, []string{"url(\")a"}},
{"url(a'\\\n)a", TTs{BadURLToken, IdentToken}, []string{"url(a'\\\n)", "a"}},
{"url(\"\n)a", TTs{BadURLToken, IdentToken}, []string{"url(\"\n)", "a"}},
{"url(a h)a", TTs{BadURLToken, IdentToken}, []string{"url(a h)", "a"}},
{" 0 && atRuleName[1] == '-' {
if i := bytes.IndexByte(atRuleName[2:], '-'); i != -1 {
atRuleName = atRuleName[i+2:] // skip vendor specific prefix
}
}
atRule := ToHash(atRuleName[1:])
first := true
skipWS := false
for {
tt, data := p.popToken(false)
if tt == LeftBraceToken && p.level == 0 {
if atRule == Font_Face || atRule == Page {
p.state = append(p.state, (*Parser).parseAtRuleDeclarationList)
} else if atRule == Document || atRule == Keyframes || atRule == Media || atRule == Supports {
p.state = append(p.state, (*Parser).parseAtRuleRuleList)
} else {
p.state = append(p.state, (*Parser).parseAtRuleUnknown)
}
return BeginAtRuleGrammar
} else if (tt == SemicolonToken || tt == RightBraceToken) && p.level == 0 || tt == ErrorToken {
p.prevEnd = (tt == RightBraceToken)
return AtRuleGrammar
} else if tt == LeftParenthesisToken || tt == LeftBraceToken || tt == LeftBracketToken || tt == FunctionToken {
p.level++
} else if tt == RightParenthesisToken || tt == RightBraceToken || tt == RightBracketToken {
p.level--
}
if first {
if tt == LeftParenthesisToken || tt == LeftBracketToken {
p.prevWS = false
}
first = false
}
if len(data) == 1 && (data[0] == ',' || data[0] == ':') {
skipWS = true
} else if p.prevWS && !skipWS && tt != RightParenthesisToken {
p.pushBuf(WhitespaceToken, wsBytes)
} else {
skipWS = false
}
if tt == LeftParenthesisToken {
skipWS = true
}
p.pushBuf(tt, data)
}
}
func (p *Parser) parseAtRuleRuleList() GrammarType {
if p.tt == RightBraceToken || p.tt == ErrorToken {
p.state = p.state[:len(p.state)-1]
return EndAtRuleGrammar
} else if p.tt == AtKeywordToken {
return p.parseAtRule()
} else {
return p.parseQualifiedRule()
}
}
func (p *Parser) parseAtRuleDeclarationList() GrammarType {
for p.tt == SemicolonToken {
p.tt, p.data = p.popToken(false)
}
if p.tt == RightBraceToken || p.tt == ErrorToken {
p.state = p.state[:len(p.state)-1]
return EndAtRuleGrammar
}
return p.parseDeclarationList()
}
func (p *Parser) parseAtRuleUnknown() GrammarType {
p.keepWS = true
if p.tt == RightBraceToken && p.level == 0 || p.tt == ErrorToken {
p.state = p.state[:len(p.state)-1]
p.keepWS = false
return EndAtRuleGrammar
}
if p.tt == LeftParenthesisToken || p.tt == LeftBraceToken || p.tt == LeftBracketToken || p.tt == FunctionToken {
p.level++
} else if p.tt == RightParenthesisToken || p.tt == RightBraceToken || p.tt == RightBracketToken {
p.level--
}
return TokenGrammar
}
func (p *Parser) parseQualifiedRule() GrammarType {
p.initBuf()
first := true
inAttrSel := false
skipWS := true
var tt TokenType
var data []byte
for {
if first {
tt, data = p.tt, p.data
p.tt = WhitespaceToken
p.data = emptyBytes
first = false
} else {
tt, data = p.popToken(false)
}
if tt == LeftBraceToken && p.level == 0 {
p.state = append(p.state, (*Parser).parseQualifiedRuleDeclarationList)
return BeginRulesetGrammar
} else if tt == ErrorToken {
p.err, p.errPos = "CSS parse error: unexpected ending in qualified rule", p.l.r.Offset()
return ErrorGrammar
} else if tt == LeftParenthesisToken || tt == LeftBraceToken || tt == LeftBracketToken || tt == FunctionToken {
p.level++
} else if tt == RightParenthesisToken || tt == RightBraceToken || tt == RightBracketToken {
p.level--
}
if len(data) == 1 && (data[0] == ',' || data[0] == '>' || data[0] == '+' || data[0] == '~') {
if data[0] == ',' {
return QualifiedRuleGrammar
}
skipWS = true
} else if p.prevWS && !skipWS && !inAttrSel {
p.pushBuf(WhitespaceToken, wsBytes)
} else {
skipWS = false
}
if tt == LeftBracketToken {
inAttrSel = true
} else if tt == RightBracketToken {
inAttrSel = false
}
p.pushBuf(tt, data)
}
}
func (p *Parser) parseQualifiedRuleDeclarationList() GrammarType {
for p.tt == SemicolonToken {
p.tt, p.data = p.popToken(false)
}
if p.tt == RightBraceToken || p.tt == ErrorToken {
p.state = p.state[:len(p.state)-1]
return EndRulesetGrammar
}
return p.parseDeclarationList()
}
func (p *Parser) parseDeclaration() GrammarType {
p.initBuf()
parse.ToLower(p.data)
ttName, dataName := p.tt, p.data
tt, data := p.popToken(false)
if tt != ColonToken {
p.l.r.Move(-len(data))
p.err, p.errPos = "CSS parse error: expected colon in declaration", p.l.r.Offset()
p.l.r.Move(len(data))
p.pushBuf(ttName, dataName)
return p.parseDeclarationError(tt, data)
}
skipWS := true
for {
tt, data := p.popToken(false)
if (tt == SemicolonToken || tt == RightBraceToken) && p.level == 0 || tt == ErrorToken {
p.prevEnd = (tt == RightBraceToken)
return DeclarationGrammar
} else if tt == LeftParenthesisToken || tt == LeftBraceToken || tt == LeftBracketToken || tt == FunctionToken {
p.level++
} else if tt == RightParenthesisToken || tt == RightBraceToken || tt == RightBracketToken {
p.level--
}
if len(data) == 1 && (data[0] == ',' || data[0] == '/' || data[0] == ':' || data[0] == '!' || data[0] == '=') {
skipWS = true
} else if (p.prevWS || p.prevComment) && !skipWS {
p.pushBuf(WhitespaceToken, wsBytes)
} else {
skipWS = false
}
p.pushBuf(tt, data)
}
}
func (p *Parser) parseDeclarationError(tt TokenType, data []byte) GrammarType {
// we're on the offending (tt,data), keep popping tokens till we reach ;, }, or EOF
p.tt, p.data = tt, data
for {
if (tt == SemicolonToken || tt == RightBraceToken) && p.level == 0 || tt == ErrorToken {
p.prevEnd = (tt == RightBraceToken)
if tt == SemicolonToken {
p.pushBuf(tt, data)
}
return ErrorGrammar
} else if tt == LeftParenthesisToken || tt == LeftBraceToken || tt == LeftBracketToken || tt == FunctionToken {
p.level++
} else if tt == RightParenthesisToken || tt == RightBraceToken || tt == RightBracketToken {
p.level--
}
if p.prevWS {
p.pushBuf(WhitespaceToken, wsBytes)
}
p.pushBuf(tt, data)
tt, data = p.popToken(false)
}
}
func (p *Parser) parseCustomProperty() GrammarType {
p.initBuf()
if tt, data := p.popToken(false); tt != ColonToken {
p.l.r.Move(-len(data))
p.err, p.errPos = "CSS parse error: expected colon in custom property", p.l.r.Offset()
p.l.r.Move(len(data))
return ErrorGrammar
}
val := []byte{}
for {
tt, data := p.l.Next()
if (tt == SemicolonToken || tt == RightBraceToken) && p.level == 0 || tt == ErrorToken {
p.prevEnd = (tt == RightBraceToken)
p.pushBuf(CustomPropertyValueToken, val)
return CustomPropertyGrammar
} else if tt == LeftParenthesisToken || tt == LeftBraceToken || tt == LeftBracketToken || tt == FunctionToken {
p.level++
} else if tt == RightParenthesisToken || tt == RightBraceToken || tt == RightBracketToken {
p.level--
}
val = append(val, data...)
}
}
parse-2.5.21/css/parse_test.go 0000664 0000000 0000000 00000022535 14117006534 0016213 0 ustar 00root root 0000000 0000000 package css
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
////////////////////////////////////////////////////////////////
func TestParse(t *testing.T) {
var parseTests = []struct {
inline bool
css string
expected string
}{
{true, " x : y ; ", "x:y;"},
{true, "color: red;", "color:red;"},
{true, "color : red;", "color:red;"},
{true, "color: red; border: 0;", "color:red;border:0;"},
{true, "color: red !important;", "color:red!important;"},
{true, "color: red ! important;", "color:red!important;"},
{true, "white-space: -moz-pre-wrap;", "white-space:-moz-pre-wrap;"},
{true, "display: -moz-inline-stack;", "display:-moz-inline-stack;"},
{true, "x: 10px / 1em;", "x:10px/1em;"},
{true, "x: 1em/1.5em \"Times New Roman\", Times, serif;", "x:1em/1.5em \"Times New Roman\",Times,serif;"},
{true, "x: hsla(100,50%, 75%, 0.5);", "x:hsla(100,50%,75%,0.5);"},
{true, "x: hsl(100,50%, 75%);", "x:hsl(100,50%,75%);"},
{true, "x: rgba(255, 238 , 221, 0.3);", "x:rgba(255,238,221,0.3);"},
{true, "x: 50vmax;", "x:50vmax;"},
{true, "color: linear-gradient(to right, black, white);", "color:linear-gradient(to right,black,white);"},
{true, "color: calc(100%/2 - 1em);", "color:calc(100%/2 - 1em);"},
{true, "color: calc(100%/2--1em);", "color:calc(100%/2--1em);"},
{false, "", ""},
{false, "@media print, screen { }", "@media print,screen{}"},
{false, "@media { @viewport ; }", "@media{@viewport;}"},
{false, "@keyframes 'diagonal-slide' { from { left: 0; top: 0; } to { left: 100px; top: 100px; } }", "@keyframes 'diagonal-slide'{from{left:0;top:0;}to{left:100px;top:100px;}}"},
{false, "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}", "@keyframes movingbox{0%{left:90%;}50%{left:10%;}100%{left:90%;}}"},
{false, ".foo { color: #fff;}", ".foo{color:#fff;}"},
{false, ".foo { ; _color: #fff;}", ".foo{_color:#fff;}"},
{false, "a { color: red; border: 0; }", "a{color:red;border:0;}"},
{false, "a { color: red; border: 0; } b { padding: 0; }", "a{color:red;border:0;}b{padding:0;}"},
{false, "/* comment */", "/* comment */"},
// extraordinary
{true, "color: red;;", "color:red;"},
{true, "margin: 10px/*comment*/50px;", "margin:10px 50px;"},
{true, "color:#c0c0c0", "color:#c0c0c0;"},
{true, "background:URL(x.png);", "background:URL(x.png);"},
{true, "filter: progid : DXImageTransform.Microsoft.BasicImage(rotation=1);", "filter:progid:DXImageTransform.Microsoft.BasicImage(rotation=1);"},
{true, "/*a*/\n/*c*/\nkey: value;", "key:value;"},
{true, "@-moz-charset;", "@-moz-charset;"},
{true, "--custom-variable: (0;) ;", "--custom-variable: (0;) ;"},
{false, "@import;@import;", "@import;@import;"},
{false, ".a .b#c, .d<.e { x:y; }", ".a .b#c,.d<.e{x:y;}"},
{false, ".a[b~=c]d { x:y; }", ".a[b~=c]d{x:y;}"},
// {false, "{x:y;}", "{x:y;}"},
{false, "a{}", "a{}"},
{false, "a,.b/*comment*/ {x:y;}", "a,.b{x:y;}"},
{false, "a,.b/*comment*/.c {x:y;}", "a,.b.c{x:y;}"},
{false, "a{x:; z:q;}", "a{x:;z:q;}"},
{false, "@font-face { x:y; }", "@font-face{x:y;}"},
{false, "a:not([controls]){x:y;}", "a:not([controls]){x:y;}"},
{false, "@document regexp('https:.*') { p { color: red; } }", "@document regexp('https:.*'){p{color:red;}}"},
{false, "@media all and ( max-width:400px ) { }", "@media all and (max-width:400px){}"},
{false, "@media (max-width:400px) { }", "@media(max-width:400px){}"},
{false, "@media (max-width:400px)", "@media(max-width:400px);"},
{false, "@font-face { ; font:x; }", "@font-face{font:x;}"},
{false, "@-moz-font-face { ; font:x; }", "@-moz-font-face{font:x;}"},
{false, "@unknown abc { {} lala }", "@unknown abc{{} lala }"},
{false, "a[x={}]{x:y;}", "a[x={}]{x:y;}"},
{false, "a[x=,]{x:y;}", "a[x=,]{x:y;}"},
{false, "a[x=+]{x:y;}", "a[x=+]{x:y;}"},
{false, ".cla .ss > #id { x:y; }", ".cla .ss>#id{x:y;}"},
{false, ".cla /*a*/ /*b*/ .ss{}", ".cla .ss{}"},
{false, "a{x:f(a(),b);}", "a{x:f(a(),b);}"},
{false, "a{x:y!z;}", "a{x:y!z;}"},
{false, "[class*=\"column\"]+[class*=\"column\"]:last-child{a:b;}", "[class*=\"column\"]+[class*=\"column\"]:last-child{a:b;}"},
{false, "@media { @viewport }", "@media{@viewport;}"},
{false, "table { @unknown }", "table{@unknown;}"},
// early endings
{false, "selector{", "selector{"},
{false, "@media{selector{", "@media{selector{"},
// bad grammar
{false, "}", "ERROR(})"},
{true, "}", "ERROR(})"},
{true, "~color:red", "ERROR(~color:red)"},
{true, "(color;red)", "ERROR((color;red))"},
{true, "color(;red)", "ERROR(color(;red))"},
{false, ".foo { *color: #fff;}", ".foo{*color:#fff;}"},
{true, "*color: red; font-size: 12pt;", "*color:red;font-size:12pt;"},
{true, "*--custom: red;", "*--custom: red;"},
{true, "_color: red; font-size: 12pt;", "_color:red;font-size:12pt;"},
{false, ".foo { baddecl } .bar { color:red; }", ".foo{ERROR(baddecl)}.bar{color:red;}"},
{false, ".foo { baddecl baddecl baddecl; height:100px } .bar { color:red; }", ".foo{ERROR(baddecl baddecl baddecl;)height:100px;}.bar{color:red;}"},
{false, ".foo { visibility: hidden;” } .bar { color:red; }", ".foo{visibility:hidden;ERROR(”)}.bar{color:red;}"},
{false, ".foo { baddecl (; color:red; }", ".foo{ERROR(baddecl (; color:red; })"},
// issues
{false, "@media print {.class{width:5px;}}", "@media print{.class{width:5px;}}"}, // #6
{false, ".class{width:calc((50% + 2em)/2 + 14px);}", ".class{width:calc((50% + 2em)/2 + 14px);}"}, // #7
{false, ".class [c=y]{}", ".class [c=y]{}"}, // tdewolff/minify#16
{false, "table{font-family:Verdana}", "table{font-family:Verdana;}"}, // tdewolff/minify#22
// go-fuzz
{false, "@-webkit-", "@-webkit-;"},
}
for _, tt := range parseTests {
t.Run(tt.css, func(t *testing.T) {
output := ""
p := NewParser(parse.NewInputString(tt.css), tt.inline)
for {
grammar, _, data := p.Next()
data = parse.Copy(data)
if grammar == ErrorGrammar {
if err := p.Err(); err != io.EOF {
data = []byte("ERROR(")
for _, val := range p.Values() {
data = append(data, val.Data...)
}
data = append(data, ")"...)
} else {
break
}
} else if grammar == AtRuleGrammar || grammar == BeginAtRuleGrammar || grammar == QualifiedRuleGrammar || grammar == BeginRulesetGrammar || grammar == DeclarationGrammar || grammar == CustomPropertyGrammar {
if grammar == DeclarationGrammar || grammar == CustomPropertyGrammar {
data = append(data, ":"...)
}
for _, val := range p.Values() {
data = append(data, val.Data...)
}
if grammar == BeginAtRuleGrammar || grammar == BeginRulesetGrammar {
data = append(data, "{"...)
} else if grammar == AtRuleGrammar || grammar == DeclarationGrammar || grammar == CustomPropertyGrammar {
data = append(data, ";"...)
} else if grammar == QualifiedRuleGrammar {
data = append(data, ","...)
}
}
output += string(data)
}
test.String(t, output, tt.expected)
})
}
// coverage
for i := 0; ; i++ {
if GrammarType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
test.T(t, Token{IdentToken, []byte("data")}.String(), "Ident('data')")
}
func TestParseError(t *testing.T) {
var parseErrorTests = []struct {
inline bool
css string
col int
}{
{false, "}", 2},
{true, "}", 1},
{false, "selector", 9},
{true, "color 0", 7},
{true, "--color 0", 9},
{true, "--custom-variable:0", 0},
}
for _, tt := range parseErrorTests {
t.Run(tt.css, func(t *testing.T) {
p := NewParser(parse.NewInputString(tt.css), tt.inline)
for {
grammar, _, _ := p.Next()
if grammar == ErrorGrammar {
if tt.col == 0 {
test.T(t, p.Err(), io.EOF)
} else if perr, ok := p.Err().(*parse.Error); ok {
test.That(t, p.HasParseError())
_, col, _ := perr.Position()
test.T(t, col, tt.col)
} else {
test.Fail(t, "bad error:", p.Err())
}
break
}
}
})
}
}
func TestParseOffset(t *testing.T) {
z := parse.NewInputString(`div{background:url(link);}`)
p := NewParser(z, false)
test.T(t, z.Offset(), 0)
_, _, _ = p.Next()
test.T(t, z.Offset(), 4) // div{
_, _, _ = p.Next()
test.T(t, z.Offset(), 25) // background:url(link);
_, _, _ = p.Next()
test.T(t, z.Offset(), 26) // }
}
////////////////////////////////////////////////////////////////
type Obj struct{}
func (*Obj) F() {}
var f1 func(*Obj)
func BenchmarkFuncPtr(b *testing.B) {
for i := 0; i < b.N; i++ {
f1 = (*Obj).F
}
}
var f2 func()
func BenchmarkMemFuncPtr(b *testing.B) {
obj := &Obj{}
for i := 0; i < b.N; i++ {
f2 = obj.F
}
}
func ExampleNewParser() {
p := NewParser(parse.NewInputString("color: red;"), true) // false because this is the content of an inline style attribute
out := ""
for {
gt, _, data := p.Next()
if gt == ErrorGrammar {
break
} else if gt == AtRuleGrammar || gt == BeginAtRuleGrammar || gt == BeginRulesetGrammar || gt == DeclarationGrammar {
out += string(data)
if gt == DeclarationGrammar {
out += ":"
}
for _, val := range p.Values() {
out += string(val.Data)
}
if gt == BeginAtRuleGrammar || gt == BeginRulesetGrammar {
out += "{"
} else if gt == AtRuleGrammar || gt == DeclarationGrammar {
out += ";"
}
} else {
out += string(data)
}
}
fmt.Println(out)
// Output: color:red;
}
parse-2.5.21/css/util.go 0000664 0000000 0000000 00000002030 14117006534 0015003 0 ustar 00root root 0000000 0000000 package css
import "github.com/tdewolff/parse/v2"
// IsIdent returns true if the bytes are a valid identifier.
func IsIdent(b []byte) bool {
l := NewLexer(parse.NewInputBytes(b))
l.consumeIdentToken()
l.r.Restore()
return l.r.Pos() == len(b)
}
// IsURLUnquoted returns true if the bytes are a valid unquoted URL.
func IsURLUnquoted(b []byte) bool {
l := NewLexer(parse.NewInputBytes(b))
l.consumeUnquotedURL()
l.r.Restore()
return l.r.Pos() == len(b)
}
// HSL2RGB converts HSL to RGB with all of range [0,1]
// from http://www.w3.org/TR/css3-color/#hsl-color
func HSL2RGB(h, s, l float64) (float64, float64, float64) {
m2 := l * (s + 1)
if l > 0.5 {
m2 = l + s - l*s
}
m1 := l*2 - m2
return hue2rgb(m1, m2, h+1.0/3.0), hue2rgb(m1, m2, h), hue2rgb(m1, m2, h-1.0/3.0)
}
func hue2rgb(m1, m2, h float64) float64 {
if h < 0.0 {
h += 1.0
}
if h > 1.0 {
h -= 1.0
}
if h*6.0 < 1.0 {
return m1 + (m2-m1)*h*6.0
} else if h*2.0 < 1.0 {
return m2
} else if h*3.0 < 2.0 {
return m1 + (m2-m1)*(2.0/3.0-h)*6.0
}
return m1
}
parse-2.5.21/css/util_test.go 0000664 0000000 0000000 00000001162 14117006534 0016047 0 ustar 00root root 0000000 0000000 package css
import (
"testing"
"github.com/tdewolff/test"
)
func TestIsIdent(t *testing.T) {
test.That(t, IsIdent([]byte("color")))
test.That(t, !IsIdent([]byte("4.5")))
}
func TestIsURLUnquoted(t *testing.T) {
test.That(t, IsURLUnquoted([]byte("http://x")))
test.That(t, !IsURLUnquoted([]byte(")")))
}
func TestHsl2Rgb(t *testing.T) {
r, g, b := HSL2RGB(0.0, 1.0, 0.5)
test.T(t, r, 1.0)
test.T(t, g, 0.0)
test.T(t, b, 0.0)
r, g, b = HSL2RGB(1.0, 1.0, 0.5)
test.T(t, r, 1.0)
test.T(t, g, 0.0)
test.T(t, b, 0.0)
r, g, b = HSL2RGB(0.66, 0.0, 1.0)
test.T(t, r, 1.0)
test.T(t, g, 1.0)
test.T(t, b, 1.0)
}
parse-2.5.21/error.go 0000664 0000000 0000000 00000002300 14117006534 0014367 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"fmt"
"io"
)
// Error is a parsing error returned by parser. It contains a message and an offset at which the error occurred.
type Error struct {
Message string
Line int
Column int
Context string
}
// NewError creates a new error
func NewError(r io.Reader, offset int, message string, a ...interface{}) *Error {
line, column, context := Position(r, offset)
if 0 < len(a) {
message = fmt.Sprintf(message, a...)
}
return &Error{
Message: message,
Line: line,
Column: column,
Context: context,
}
}
// NewErrorLexer creates a new error from an active Lexer.
func NewErrorLexer(l *Input, message string, a ...interface{}) *Error {
r := bytes.NewBuffer(l.Bytes())
offset := l.Offset()
return NewError(r, offset, message, a...)
}
// Position returns the line, column, and context of the error.
// Context is the entire line at which the error occurred.
func (e *Error) Position() (int, int, string) {
return e.Line, e.Column, e.Context
}
// Error returns the error string, containing the context and line + column number.
func (e *Error) Error() string {
return fmt.Sprintf("%s on line %d and column %d\n%s", e.Message, e.Line, e.Column, e.Context)
}
parse-2.5.21/error_test.go 0000664 0000000 0000000 00000002007 14117006534 0015432 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"testing"
"github.com/tdewolff/test"
)
func TestError(t *testing.T) {
err := NewError(bytes.NewBufferString("buffer"), 3, "message")
line, column, context := err.Position()
test.T(t, line, 1, "line")
test.T(t, column, 4, "column")
test.T(t, "\n"+context, "\n 1: buffer\n ^", "context")
test.T(t, err.Error(), "message on line 1 and column 4\n 1: buffer\n ^", "error")
}
func TestErrorLexer(t *testing.T) {
l := NewInputString("buffer")
l.Move(3)
err := NewErrorLexer(l, "message")
line, column, context := err.Position()
test.T(t, line, 1, "line")
test.T(t, column, 4, "column")
test.T(t, "\n"+context, "\n 1: buffer\n ^", "context")
test.T(t, err.Error(), "message on line 1 and column 4\n 1: buffer\n ^", "error")
}
func TestErrorMessages(t *testing.T) {
err := NewError(bytes.NewBufferString("buffer"), 3, "message %d", 5)
test.T(t, err.Error(), "message 5 on line 1 and column 4\n 1: buffer\n ^", "error")
}
parse-2.5.21/go.mod 0000664 0000000 0000000 00000000126 14117006534 0014021 0 ustar 00root root 0000000 0000000 module github.com/tdewolff/parse/v2
go 1.13
require github.com/tdewolff/test v1.0.6
parse-2.5.21/go.sum 0000664 0000000 0000000 00000000247 14117006534 0014052 0 ustar 00root root 0000000 0000000 github.com/tdewolff/test v1.0.6 h1:76mzYJQ83Op284kMT+63iCNCI7NEERsIN8dLM+RiKr4=
github.com/tdewolff/test v1.0.6/go.mod h1:6DAvZliBAAnD7rhVgwaM7DE5/d9NMOAJ09SqYqeK4QE=
parse-2.5.21/html/ 0000775 0000000 0000000 00000000000 14117006534 0013660 5 ustar 00root root 0000000 0000000 parse-2.5.21/html/README.md 0000664 0000000 0000000 00000003656 14117006534 0015151 0 ustar 00root root 0000000 0000000 # HTML [](https://pkg.go.dev/github.com/tdewolff/parse/v2/html?tab=doc)
This package is an HTML5 lexer written in [Go][1]. It follows the specification at [The HTML syntax](http://www.w3.org/TR/html5/syntax.html). The lexer takes an io.Reader and converts it into tokens until the EOF.
## Installation
Run the following command
go get -u github.com/tdewolff/parse/v2/html
or add the following import and run project with `go get`
import "github.com/tdewolff/parse/v2/html"
## Lexer
### Usage
The following initializes a new Lexer with io.Reader `r`:
``` go
l := html.NewLexer(parse.NewInput(r))
```
To tokenize until EOF an error, use:
``` go
for {
tt, data := l.Next()
switch tt {
case html.ErrorToken:
// error or EOF set in l.Err()
return
case html.StartTagToken:
// ...
for {
ttAttr, dataAttr := l.Next()
if ttAttr != html.AttributeToken {
break
}
// ...
}
// ...
}
}
```
All tokens:
``` go
ErrorToken TokenType = iota // extra token when errors occur
CommentToken
DoctypeToken
StartTagToken
StartTagCloseToken
StartTagVoidToken
EndTagToken
AttributeToken
TextToken
```
### Examples
``` go
package main
import (
"os"
"github.com/tdewolff/parse/v2/html"
)
// Tokenize HTML from stdin.
func main() {
l := html.NewLexer(parse.NewInput(os.Stdin))
for {
tt, data := l.Next()
switch tt {
case html.ErrorToken:
if l.Err() != io.EOF {
fmt.Println("Error on line", l.Line(), ":", l.Err())
}
return
case html.StartTagToken:
fmt.Println("Tag", string(data))
for {
ttAttr, dataAttr := l.Next()
if ttAttr != html.AttributeToken {
break
}
key := dataAttr
val := l.AttrVal()
fmt.Println("Attribute", string(key), "=", string(val))
}
// ...
}
}
}
```
## License
Released under the [MIT license](https://github.com/tdewolff/parse/blob/master/LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/html/hash.go 0000664 0000000 0000000 00000003700 14117006534 0015132 0 ustar 00root root 0000000 0000000 package html
// generated by hasher -type=Hash -file=hash.go; DO NOT EDIT, except for adding more constants to the list and rerun go generate
// uses github.com/tdewolff/hasher
//go:generate hasher -type=Hash -file=hash.go
// Hash defines perfect hashes for a predefined list of strings
type Hash uint32
// Unique hash definitions to be used instead of strings
const (
Iframe Hash = 0x6 // iframe
Math Hash = 0x604 // math
Plaintext Hash = 0x1e09 // plaintext
Script Hash = 0xa06 // script
Style Hash = 0x1405 // style
Svg Hash = 0x1903 // svg
Textarea Hash = 0x2308 // textarea
Title Hash = 0xf05 // title
Xmp Hash = 0x1c03 // xmp
)
// String returns the hash' name.
func (i Hash) String() string {
start := uint32(i >> 8)
n := uint32(i & 0xff)
if start+n > uint32(len(_Hash_text)) {
return ""
}
return _Hash_text[start : start+n]
}
// ToHash returns the hash whose name is s. It returns zero if there is no
// such hash. It is case sensitive.
func ToHash(s []byte) Hash {
if len(s) == 0 || len(s) > _Hash_maxLen {
return 0
}
h := uint32(_Hash_hash0)
for i := 0; i < len(s); i++ {
h ^= uint32(s[i])
h *= 16777619
}
if i := _Hash_table[h&uint32(len(_Hash_table)-1)]; int(i&0xff) == len(s) {
t := _Hash_text[i>>8 : i>>8+i&0xff]
for i := 0; i < len(s); i++ {
if t[i] != s[i] {
goto NEXT
}
}
return i
}
NEXT:
if i := _Hash_table[(h>>16)&uint32(len(_Hash_table)-1)]; int(i&0xff) == len(s) {
t := _Hash_text[i>>8 : i>>8+i&0xff]
for i := 0; i < len(s); i++ {
if t[i] != s[i] {
return 0
}
}
return i
}
return 0
}
const _Hash_hash0 = 0x9acb0442
const _Hash_maxLen = 9
const _Hash_text = "iframemathscriptitlestylesvgxmplaintextarea"
var _Hash_table = [1 << 4]Hash{
0x0: 0x2308, // textarea
0x2: 0x6, // iframe
0x4: 0xf05, // title
0x5: 0x1e09, // plaintext
0x7: 0x1405, // style
0x8: 0x604, // math
0x9: 0xa06, // script
0xa: 0x1903, // svg
0xb: 0x1c03, // xmp
}
parse-2.5.21/html/lex.go 0000664 0000000 0000000 00000026453 14117006534 0015011 0 ustar 00root root 0000000 0000000 // Package html is an HTML5 lexer following the specifications at http://www.w3.org/TR/html5/syntax.html.
package html
import (
"strconv"
"github.com/tdewolff/parse/v2"
)
// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint32
// TokenType values.
const (
ErrorToken TokenType = iota // extra token when errors occur
CommentToken
DoctypeToken
StartTagToken
StartTagCloseToken
StartTagVoidToken
EndTagToken
AttributeToken
TextToken
SvgToken
MathToken
)
// String returns the string representation of a TokenType.
func (tt TokenType) String() string {
switch tt {
case ErrorToken:
return "Error"
case CommentToken:
return "Comment"
case DoctypeToken:
return "Doctype"
case StartTagToken:
return "StartTag"
case StartTagCloseToken:
return "StartTagClose"
case StartTagVoidToken:
return "StartTagVoid"
case EndTagToken:
return "EndTag"
case AttributeToken:
return "Attribute"
case TextToken:
return "Text"
case SvgToken:
return "Svg"
case MathToken:
return "Math"
}
return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}
////////////////////////////////////////////////////////////////
// Lexer is the state for the lexer.
type Lexer struct {
r *parse.Input
err error
rawTag Hash
inTag bool
text []byte
attrVal []byte
}
// NewLexer returns a new Lexer for a given io.Reader.
func NewLexer(r *parse.Input) *Lexer {
return &Lexer{
r: r,
}
}
// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
func (l *Lexer) Err() error {
if l.err != nil {
return l.err
}
return l.r.Err()
}
// Text returns the textual representation of a token. This excludes delimiters and additional leading/trailing characters.
func (l *Lexer) Text() []byte {
return l.text
}
// AttrVal returns the attribute value when an AttributeToken was returned from Next.
func (l *Lexer) AttrVal() []byte {
return l.attrVal
}
// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (l *Lexer) Next() (TokenType, []byte) {
l.text = nil
var c byte
if l.inTag {
l.attrVal = nil
for { // before attribute name state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
l.r.Move(1)
continue
}
break
}
if c == 0 && l.r.Err() != nil {
return ErrorToken, nil
} else if c != '>' && (c != '/' || l.r.Peek(1) != '>') {
return AttributeToken, l.shiftAttribute()
}
l.r.Skip()
l.inTag = false
if c == '/' {
l.r.Move(2)
return StartTagVoidToken, l.r.Shift()
}
l.r.Move(1)
return StartTagCloseToken, l.r.Shift()
}
if l.rawTag != 0 {
if rawText := l.shiftRawText(); len(rawText) > 0 {
l.rawTag = 0
return TextToken, rawText
}
l.rawTag = 0
}
for {
c = l.r.Peek(0)
if c == '<' {
c = l.r.Peek(1)
isEndTag := c == '/' && l.r.Peek(2) != '>' && (l.r.Peek(2) != 0 || l.r.PeekErr(2) == nil)
if l.r.Pos() > 0 {
if isEndTag || 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || c == '!' || c == '?' {
// return currently buffered texttoken so that we can return tag next iteration
l.text = l.r.Shift()
return TextToken, l.text
}
} else if isEndTag {
l.r.Move(2)
// only endtags that are not followed by > or EOF arrive here
if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
return CommentToken, l.shiftBogusComment()
}
return EndTagToken, l.shiftEndTag()
} else if 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' {
l.r.Move(1)
l.inTag = true
return l.shiftStartTag()
} else if c == '!' {
l.r.Move(2)
return l.readMarkup()
} else if c == '?' {
l.r.Move(1)
return CommentToken, l.shiftBogusComment()
}
} else if c == 0 && l.r.Err() != nil {
if l.r.Pos() > 0 {
l.text = l.r.Shift()
return TextToken, l.text
}
return ErrorToken, nil
}
l.r.Move(1)
}
}
////////////////////////////////////////////////////////////////
// The following functions follow the specifications at https://html.spec.whatwg.org/multipage/parsing.html
func (l *Lexer) shiftRawText() []byte {
if l.rawTag == Plaintext {
for {
if l.r.Peek(0) == 0 && l.r.Err() != nil {
return l.r.Shift()
}
l.r.Move(1)
}
} else { // RCDATA, RAWTEXT and SCRIPT
for {
c := l.r.Peek(0)
if c == '<' {
if l.r.Peek(1) == '/' {
mark := l.r.Pos()
l.r.Move(2)
for {
if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
break
}
l.r.Move(1)
}
if h := ToHash(parse.ToLower(parse.Copy(l.r.Lexeme()[mark+2:]))); h == l.rawTag { // copy so that ToLower doesn't change the case of the underlying slice
l.r.Rewind(mark)
return l.r.Shift()
}
} else if l.rawTag == Script && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
l.r.Move(4)
inScript := false
for {
c := l.r.Peek(0)
if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
l.r.Move(3)
break
} else if c == '<' {
isEnd := l.r.Peek(1) == '/'
if isEnd {
l.r.Move(2)
} else {
l.r.Move(1)
}
mark := l.r.Pos()
for {
if c = l.r.Peek(0); !('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
break
}
l.r.Move(1)
}
if h := ToHash(parse.ToLower(parse.Copy(l.r.Lexeme()[mark:]))); h == Script { // copy so that ToLower doesn't change the case of the underlying slice
if !isEnd {
inScript = true
} else {
if !inScript {
l.r.Rewind(mark - 2)
return l.r.Shift()
}
inScript = false
}
}
} else if c == 0 && l.r.Err() != nil {
return l.r.Shift()
} else {
l.r.Move(1)
}
}
} else {
l.r.Move(1)
}
} else if c == 0 && l.r.Err() != nil {
return l.r.Shift()
} else {
l.r.Move(1)
}
}
}
}
func (l *Lexer) readMarkup() (TokenType, []byte) {
if l.at('-', '-') {
l.r.Move(2)
for {
if l.r.Peek(0) == 0 && l.r.Err() != nil {
l.text = l.r.Lexeme()[4:]
return CommentToken, l.r.Shift()
} else if l.at('-', '-', '>') {
l.text = l.r.Lexeme()[4:]
l.r.Move(3)
return CommentToken, l.r.Shift()
} else if l.at('-', '-', '!', '>') {
l.text = l.r.Lexeme()[4:]
l.r.Move(4)
return CommentToken, l.r.Shift()
}
l.r.Move(1)
}
} else if l.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
l.r.Move(7)
for {
if l.r.Peek(0) == 0 && l.r.Err() != nil {
l.text = l.r.Lexeme()[9:]
return TextToken, l.r.Shift()
} else if l.at(']', ']', '>') {
l.text = l.r.Lexeme()[9:]
l.r.Move(3)
return TextToken, l.r.Shift()
}
l.r.Move(1)
}
} else {
if l.atCaseInsensitive('d', 'o', 'c', 't', 'y', 'p', 'e') {
l.r.Move(7)
if l.r.Peek(0) == ' ' {
l.r.Move(1)
}
for {
if c := l.r.Peek(0); c == '>' || c == 0 && l.r.Err() != nil {
l.text = l.r.Lexeme()[9:]
if c == '>' {
l.r.Move(1)
}
return DoctypeToken, l.r.Shift()
}
l.r.Move(1)
}
}
}
return CommentToken, l.shiftBogusComment()
}
func (l *Lexer) shiftBogusComment() []byte {
for {
c := l.r.Peek(0)
if c == '>' {
l.text = l.r.Lexeme()[2:]
l.r.Move(1)
return l.r.Shift()
} else if c == 0 && l.r.Err() != nil {
l.text = l.r.Lexeme()[2:]
return l.r.Shift()
}
l.r.Move(1)
}
}
func (l *Lexer) shiftStartTag() (TokenType, []byte) {
for {
if c := l.r.Peek(0); c == ' ' || c == '>' || c == '/' && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
break
}
l.r.Move(1)
}
l.text = parse.ToLower(l.r.Lexeme()[1:])
if h := ToHash(l.text); h == Textarea || h == Title || h == Style || h == Xmp || h == Iframe || h == Script || h == Plaintext || h == Svg || h == Math {
if h == Svg || h == Math {
data := l.shiftXML(h)
if l.err != nil {
return ErrorToken, nil
}
l.inTag = false
if h == Svg {
return SvgToken, data
}
return MathToken, data
}
l.rawTag = h
}
return StartTagToken, l.r.Shift()
}
func (l *Lexer) shiftAttribute() []byte {
nameStart := l.r.Pos()
var c byte
for { // attribute name state
if c = l.r.Peek(0); c == ' ' || c == '=' || c == '>' || c == '/' && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
break
}
l.r.Move(1)
}
nameEnd := l.r.Pos()
for { // after attribute name state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
l.r.Move(1)
continue
}
break
}
if c == '=' {
l.r.Move(1)
for { // before attribute value state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
l.r.Move(1)
continue
}
break
}
attrPos := l.r.Pos()
delim := c
if delim == '"' || delim == '\'' { // attribute value single- and double-quoted state
l.r.Move(1)
for {
c := l.r.Peek(0)
if c == delim {
l.r.Move(1)
break
} else if c == 0 && l.r.Err() != nil {
break
}
l.r.Move(1)
}
} else { // attribute value unquoted state
for {
if c := l.r.Peek(0); c == ' ' || c == '>' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == 0 && l.r.Err() != nil {
break
}
l.r.Move(1)
}
}
l.attrVal = l.r.Lexeme()[attrPos:]
} else {
l.r.Rewind(nameEnd)
l.attrVal = nil
}
l.text = parse.ToLower(l.r.Lexeme()[nameStart:nameEnd])
return l.r.Shift()
}
func (l *Lexer) shiftEndTag() []byte {
for {
c := l.r.Peek(0)
if c == '>' {
l.text = l.r.Lexeme()[2:]
l.r.Move(1)
break
} else if c == 0 && l.r.Err() != nil {
l.text = l.r.Lexeme()[2:]
break
}
l.r.Move(1)
}
end := len(l.text)
for end > 0 {
if c := l.text[end-1]; c == ' ' || c == '\t' || c == '\n' || c == '\r' {
end--
continue
}
break
}
l.text = l.text[:end]
return parse.ToLower(l.r.Shift())
}
// shiftXML parses the content of a svg or math tag according to the XML 1.1 specifications, including the tag itself.
// So far we have already parsed `' {
l.r.Move(1)
break
} else if c == 0 {
if l.r.Err() == nil {
l.err = parse.NewErrorLexer(l.r, "HTML parse error: unexpected NULL character")
}
return l.r.Shift()
}
l.r.Move(1)
}
return l.r.Shift()
}
////////////////////////////////////////////////////////////////
func (l *Lexer) at(b ...byte) bool {
for i, c := range b {
if l.r.Peek(i) != c {
return false
}
}
return true
}
func (l *Lexer) atCaseInsensitive(b ...byte) bool {
for i, c := range b {
if l.r.Peek(i) != c && (l.r.Peek(i)+('a'-'A')) != c {
return false
}
}
return true
}
parse-2.5.21/html/lex_test.go 0000664 0000000 0000000 00000024243 14117006534 0016043 0 ustar 00root root 0000000 0000000 package html
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type TTs []TokenType
func TestTokens(t *testing.T) {
var tokenTests = []struct {
html string
expected []TokenType
}{
{"", TTs{StartTagToken, StartTagCloseToken, EndTagToken}},
{"", TTs{StartTagToken, StartTagVoidToken}},
{"", TTs{CommentToken}},
{"", TTs{CommentToken}},
{"
text
", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"", TTs{StartTagToken, AttributeToken, StartTagVoidToken}},
{"", TTs{StartTagToken, AttributeToken, AttributeToken, StartTagVoidToken}},
{"", TTs{StartTagToken, AttributeToken, AttributeToken, AttributeToken, AttributeToken, StartTagVoidToken}},
{"", TTs{DoctypeToken}},
{"", TTs{DoctypeToken}},
{"", TTs{CommentToken}},
{"", TTs{CommentToken}},
{"", TTs{CommentToken}},
{"< ", TTs{TextToken}},
{"", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"<p></p>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"</plaintext>", TTs{StartTagToken, StartTagCloseToken, TextToken}},
{"<script></script>", TTs{StartTagToken, StartTagCloseToken, EndTagToken}},
{"<script>var x='</script>';</script>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken, TextToken, EndTagToken}},
{"<script><!--var x='</script>';--></script>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken, TextToken, EndTagToken}},
{"<script><!--var x='<script></script>';--></script>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"<script><!--var x='<script>';--></script>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"<![CDATA[ test ]]>", TTs{TextToken}},
{"<svg>text</svg>", TTs{SvgToken}},
{"<math>text</math gibberish>", TTs{MathToken}},
{`<svg>text<x a="</svg>"></x></svg>`, TTs{SvgToken}},
{"<a><svg>text</svg></a>", TTs{StartTagToken, StartTagCloseToken, SvgToken, EndTagToken}},
// early endings
{"<!-- comment", TTs{CommentToken}},
{"<? bogus comment", TTs{CommentToken}},
{"<foo", TTs{StartTagToken}},
{"</foo", TTs{EndTagToken}},
{"<foo x", TTs{StartTagToken, AttributeToken}},
{"<foo x=", TTs{StartTagToken, AttributeToken}},
{"<foo x='", TTs{StartTagToken, AttributeToken}},
{"<foo x=''", TTs{StartTagToken, AttributeToken}},
{"<!DOCTYPE note SYSTEM", TTs{DoctypeToken}},
{"<![CDATA[ test", TTs{TextToken}},
{"<script>", TTs{StartTagToken, StartTagCloseToken}},
{"<script><!--", TTs{StartTagToken, StartTagCloseToken, TextToken}},
{"<script><!--var x='<script></script>';-->", TTs{StartTagToken, StartTagCloseToken, TextToken}},
// NULL
{"foo\x00bar", TTs{TextToken}},
{"<\x00foo>", TTs{TextToken}},
{"<foo\x00>", TTs{StartTagToken, StartTagCloseToken}},
{"</\x00bogus>", TTs{CommentToken}},
{"</foo\x00>", TTs{EndTagToken}},
{"<plaintext>\x00</plaintext>", TTs{StartTagToken, StartTagCloseToken, TextToken}},
{"<script>\x00</script>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"<!--\x00-->", TTs{CommentToken}},
{"<![CDATA[\x00]]>", TTs{TextToken}},
{"<!doctype\x00>", TTs{DoctypeToken}},
{"<?bogus\x00>", TTs{CommentToken}},
{"<?bogus\x00>", TTs{CommentToken}},
// go-fuzz
{"</>", TTs{TextToken}},
}
for _, tt := range tokenTests {
t.Run(tt.html, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.html))
i := 0
tokens := []TokenType{}
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
break
}
tokens = append(tokens, token)
i++
}
test.T(t, tokens, tt.expected, "token types must match")
})
}
// coverage
for i := 0; ; i++ {
if TokenType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
}
func TestTags(t *testing.T) {
var tagTests = []struct {
html string
expected string
}{
{"<foo:bar.qux-norf/>", "foo:bar.qux-norf"},
{"<foo?bar/qux>", "foo?bar/qux"},
{"<!DOCTYPE note SYSTEM \"Note.dtd\">", " note SYSTEM \"Note.dtd\""},
{"</foo >", "foo"},
// early endings
{"<foo ", "foo"},
}
for _, tt := range tagTests {
t.Run(tt.html, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.html))
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.Fail(t, "when error occurred we must be at the end")
break
} else if token == StartTagToken || token == EndTagToken || token == DoctypeToken {
test.String(t, string(l.Text()), tt.expected)
break
}
}
})
}
}
func TestAttributes(t *testing.T) {
var attributeTests = []struct {
attr string
expected []string
}{
{"<foo a=\"b\" />", []string{"a", "\"b\""}},
{"<foo \nchecked \r\n value\r=\t'=/>\"' />", []string{"checked", "", "value", "'=/>\"'"}},
{"<foo bar=\" a \n\t\r b \" />", []string{"bar", "\" a \n\t\r b \""}},
{"<foo a/>", []string{"a", ""}},
{"<foo /=/>", []string{"/", "/"}},
// early endings
{"<foo x", []string{"x", ""}},
{"<foo x=", []string{"x", ""}},
{"<foo x='", []string{"x", "'"}},
// NULL
{"<foo \x00>", []string{"\x00", ""}},
{"<foo \x00=\x00>", []string{"\x00", "\x00"}},
{"<foo \x00='\x00'>", []string{"\x00", "'\x00'"}},
}
for _, tt := range attributeTests {
t.Run(tt.attr, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.attr))
i := 0
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
} else if token == AttributeToken {
test.That(t, i+1 < len(tt.expected), "index", i+1, "must not exceed expected attributes size", len(tt.expected))
if i+1 < len(tt.expected) {
test.String(t, string(l.Text()), tt.expected[i], "attribute keys must match")
test.String(t, string(l.AttrVal()), tt.expected[i+1], "attribute keys must match")
i += 2
}
}
}
})
}
}
func TestErrors(t *testing.T) {
var errorTests = []struct {
html string
col int
}{
{"<svg>\x00</svg>", 6},
{"<svg></svg\x00>", 11},
}
for _, tt := range errorTests {
t.Run(tt.html, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.html))
for {
token, _ := l.Next()
if token == ErrorToken {
if tt.col == 0 {
test.T(t, l.Err(), io.EOF)
} else if perr, ok := l.Err().(*parse.Error); ok {
_, col, _ := perr.Position()
test.T(t, col, tt.col)
} else {
test.Fail(t, "bad error:", l.Err())
}
break
}
}
})
}
}
func TestTextAndAttrVal(t *testing.T) {
l := NewLexer(parse.NewInputString(`<div attr="val" >text<!--comment--><!DOCTYPE doctype><![CDATA[cdata]]><script>js</script><svg>image</svg>`))
_, data := l.Next()
test.Bytes(t, data, []byte("<div"))
test.Bytes(t, l.Text(), []byte("div"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte(` attr="val"`))
test.Bytes(t, l.Text(), []byte("attr"))
test.Bytes(t, l.AttrVal(), []byte(`"val"`))
_, data = l.Next()
test.Bytes(t, data, []byte(">"))
test.Bytes(t, l.Text(), nil)
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("text"))
test.Bytes(t, l.Text(), []byte("text"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<!--comment-->"))
test.Bytes(t, l.Text(), []byte("comment"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<!DOCTYPE doctype>"))
test.Bytes(t, l.Text(), []byte(" doctype"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<![CDATA[cdata]]>"))
test.Bytes(t, l.Text(), []byte("cdata"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<script"))
test.Bytes(t, l.Text(), []byte("script"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte(">"))
test.Bytes(t, l.Text(), nil)
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("js"))
test.Bytes(t, l.Text(), nil)
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("</script>"))
test.Bytes(t, l.Text(), []byte("script"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<svg>image</svg>"))
test.Bytes(t, l.Text(), []byte("svg"))
test.Bytes(t, l.AttrVal(), nil)
}
func TestOffset(t *testing.T) {
z := parse.NewInputString(`<div attr="val">text</div>`)
l := NewLexer(z)
test.T(t, z.Offset(), 0)
_, _ = l.Next()
test.T(t, z.Offset(), 4) // <div
_, _ = l.Next()
test.T(t, z.Offset(), 15) // attr="val"
_, _ = l.Next()
test.T(t, z.Offset(), 16) // >
_, _ = l.Next()
test.T(t, z.Offset(), 20) // text
_, _ = l.Next()
test.T(t, z.Offset(), 26) // </div>
}
////////////////////////////////////////////////////////////////
var J int
var ss = [][]byte{
[]byte(" style"),
[]byte("style"),
[]byte(" \r\n\tstyle"),
[]byte(" style"),
[]byte(" x"),
[]byte("x"),
}
func BenchmarkWhitespace1(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, s := range ss {
j := 0
for {
if c := s[j]; c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
j++
} else {
break
}
}
J += j
}
}
}
func BenchmarkWhitespace2(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, s := range ss {
j := 0
for {
if c := s[j]; c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' {
j++
continue
}
break
}
J += j
}
}
}
func BenchmarkWhitespace3(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, s := range ss {
j := 0
for {
if c := s[j]; c != ' ' && c != '\t' && c != '\n' && c != '\r' && c != '\f' {
break
}
j++
}
J += j
}
}
}
////////////////////////////////////////////////////////////////
func ExampleNewLexer() {
l := NewLexer(parse.NewInputString("<span class='user'>John Doe</span>"))
out := ""
for {
tt, data := l.Next()
if tt == ErrorToken {
break
}
out += string(data)
}
fmt.Println(out)
// Output: <span class='user'>John Doe</span>
}
parse-2.5.21/html/util.go 0000664 0000000 0000000 00000006176 14117006534 0015176 0 ustar 00root root 0000000 0000000 package html
var (
singleQuoteEntityBytes = []byte("'")
doubleQuoteEntityBytes = []byte(""")
)
// EscapeAttrVal returns the escaped attribute value bytes without quotes.
func EscapeAttrVal(buf *[]byte, orig, b []byte, isXML bool) []byte {
singles := 0
doubles := 0
unquoted := true
entities := false
for _, c := range b {
if charTable[c] {
unquoted = false
if c == '"' {
doubles++
} else if c == '\'' {
singles++
}
}
}
if unquoted && !isXML {
return b
} else if !entities && len(orig) == len(b)+2 && (singles == 0 && orig[0] == '\'' || doubles == 0 && orig[0] == '"') {
return orig
}
n := len(b) + 2
var quote byte
var escapedQuote []byte
if singles >= doubles || isXML {
n += doubles * 4
quote = '"'
escapedQuote = doubleQuoteEntityBytes
} else {
n += singles * 4
quote = '\''
escapedQuote = singleQuoteEntityBytes
}
if n > cap(*buf) {
*buf = make([]byte, 0, n) // maximum size, not actual size
}
t := (*buf)[:n] // maximum size, not actual size
t[0] = quote
j := 1
start := 0
for i, c := range b {
if c == quote {
j += copy(t[j:], b[start:i])
j += copy(t[j:], escapedQuote)
start = i + 1
}
}
j += copy(t[j:], b[start:])
t[j] = quote
return t[:j+1]
}
var charTable = [256]bool{
// ASCII
false, false, false, false, false, false, false, false,
false, true, true, false, true, true, false, false, // tab, line feed, form feed, carriage return
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
true, false, true, false, false, false, false, true, // space, "), '
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, true, true, true, false, // <, =, >
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
true, false, false, false, false, false, false, false, // `
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
// non-ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
}
parse-2.5.21/html/util_test.go 0000664 0000000 0000000 00000002447 14117006534 0016232 0 ustar 00root root 0000000 0000000 package html
import (
"testing"
"github.com/tdewolff/test"
)
func TestEscapeAttrVal(t *testing.T) {
var escapeAttrValTests = []struct {
attrVal string
expected string
}{
{`xyz`, `xyz`},
{``, ``},
{`x/z`, `x/z`},
{`x'z`, `"x'z"`},
{`x"z`, `'x"z'`},
{`'x"z'`, `'x"z'`},
{`'x'"'z'`, `"x'"'z"`},
{`"x"'"z"`, `'x"'"z'`},
{`"x'z"`, `"x'z"`},
{`'x'z'`, `"x'z"`},
{`a'b=""`, `'a'b=""'`},
{`x<z`, `"x<z"`},
{`'x"'"z'`, `'x"'"z'`},
}
var buf []byte
for _, tt := range escapeAttrValTests {
t.Run(tt.attrVal, func(t *testing.T) {
b := []byte(tt.attrVal)
orig := b
if len(b) > 1 && (b[0] == '"' || b[0] == '\'') && b[0] == b[len(b)-1] {
b = b[1 : len(b)-1]
}
val := EscapeAttrVal(&buf, orig, b, false)
test.String(t, string(val), tt.expected)
})
}
}
func TestEscapeAttrValXML(t *testing.T) {
var escapeAttrValTests = []struct {
attrVal string
expected string
}{
{`xyz`, `"xyz"`},
{``, `""`},
}
var buf []byte
for _, tt := range escapeAttrValTests {
t.Run(tt.attrVal, func(t *testing.T) {
b := []byte(tt.attrVal)
orig := b
if len(b) > 1 && (b[0] == '"' || b[0] == '\'') && b[0] == b[len(b)-1] {
b = b[1 : len(b)-1]
}
val := EscapeAttrVal(&buf, orig, b, true)
test.String(t, string(val), tt.expected)
})
}
}
parse-2.5.21/input.go 0000664 0000000 0000000 00000010072 14117006534 0014402 0 ustar 00root root 0000000 0000000 package parse
import (
"io"
"io/ioutil"
)
var nullBuffer = []byte{0}
// Input is a buffered reader that allows peeking forward and shifting, taking an io.Input.
// It keeps data in-memory until Free, taking a byte length, is called to move beyond the data.
type Input struct {
buf []byte
pos int // index in buf
start int // index in buf
err error
restore func()
}
// NewInput returns a new Input for a given io.Input and uses ioutil.ReadAll to read it into a byte slice.
// If the io.Input implements Bytes, that is used instead. It will append a NULL at the end of the buffer.
func NewInput(r io.Reader) *Input {
var b []byte
if r != nil {
if buffer, ok := r.(interface {
Bytes() []byte
}); ok {
b = buffer.Bytes()
} else {
var err error
b, err = ioutil.ReadAll(r)
if err != nil {
return &Input{
buf: nullBuffer,
err: err,
}
}
}
}
return NewInputBytes(b)
}
// NewInputString returns a new Input for a given string and appends NULL at the end.
func NewInputString(s string) *Input {
return NewInputBytes([]byte(s))
}
// NewInputBytes returns a new Input for a given byte slice and appends NULL at the end.
// To avoid reallocation, make sure the capacity has room for one more byte.
func NewInputBytes(b []byte) *Input {
z := &Input{
buf: b,
}
n := len(b)
if n == 0 {
z.buf = nullBuffer
} else {
// Append NULL to buffer, but try to avoid reallocation
if cap(b) > n {
// Overwrite next byte but restore when done
b = b[:n+1]
c := b[n]
b[n] = 0
z.buf = b
z.restore = func() {
b[n] = c
}
} else {
z.buf = append(b, 0)
}
}
return z
}
// Restore restores the replaced byte past the end of the buffer by NULL.
func (z *Input) Restore() {
if z.restore != nil {
z.restore()
z.restore = nil
}
}
// Err returns the error returned from io.Input or io.EOF when the end has been reached.
func (z *Input) Err() error {
return z.PeekErr(0)
}
// PeekErr returns the error at position pos. When pos is zero, this is the same as calling Err().
func (z *Input) PeekErr(pos int) error {
if z.err != nil {
return z.err
} else if z.pos+pos >= len(z.buf)-1 {
return io.EOF
}
return nil
}
// Peek returns the ith byte relative to the end position.
// Peek returns 0 when an error has occurred, Err returns the erroz.
func (z *Input) Peek(pos int) byte {
pos += z.pos
return z.buf[pos]
}
// PeekRune returns the rune and rune length of the ith byte relative to the end position.
func (z *Input) PeekRune(pos int) (rune, int) {
// from unicode/utf8
c := z.Peek(pos)
if c < 0xC0 || z.Peek(pos+1) == 0 {
return rune(c), 1
} else if c < 0xE0 || z.Peek(pos+2) == 0 {
return rune(c&0x1F)<<6 | rune(z.Peek(pos+1)&0x3F), 2
} else if c < 0xF0 || z.Peek(pos+3) == 0 {
return rune(c&0x0F)<<12 | rune(z.Peek(pos+1)&0x3F)<<6 | rune(z.Peek(pos+2)&0x3F), 3
}
return rune(c&0x07)<<18 | rune(z.Peek(pos+1)&0x3F)<<12 | rune(z.Peek(pos+2)&0x3F)<<6 | rune(z.Peek(pos+3)&0x3F), 4
}
// Move advances the position.
func (z *Input) Move(n int) {
z.pos += n
}
// Pos returns a mark to which can be rewinded.
func (z *Input) Pos() int {
return z.pos - z.start
}
// Rewind rewinds the position to the given position.
func (z *Input) Rewind(pos int) {
z.pos = z.start + pos
}
// Lexeme returns the bytes of the current selection.
func (z *Input) Lexeme() []byte {
return z.buf[z.start:z.pos:z.pos]
}
// Skip collapses the position to the end of the selection.
func (z *Input) Skip() {
z.start = z.pos
}
// Shift returns the bytes of the current selection and collapses the position to the end of the selection.
func (z *Input) Shift() []byte {
b := z.buf[z.start:z.pos:z.pos]
z.start = z.pos
return b
}
// Offset returns the character position in the buffez.
func (z *Input) Offset() int {
return z.pos
}
// Bytes returns the underlying buffez.
func (z *Input) Bytes() []byte {
return z.buf[: len(z.buf)-1 : len(z.buf)-1]
}
// Len returns the length of the underlying buffez.
func (z *Input) Len() int {
return len(z.buf) - 1
}
// Reset resets position to the underlying buffez.
func (z *Input) Reset() {
z.start = 0
z.pos = 0
}
parse-2.5.21/input_test.go 0000664 0000000 0000000 00000010136 14117006534 0015442 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"io"
"testing"
"github.com/tdewolff/test"
)
func TestInput(t *testing.T) {
s := `Lorem ipsum dolor sit amet, consectetur adipiscing elit.`
z := NewInput(bytes.NewBufferString(s))
test.Bytes(t, z.Bytes(), []byte(s), "bytes match original buffer")
test.T(t, z.err, nil, "buffer has no error")
test.T(t, z.Err(), nil, "buffer is at EOF but must not return EOF until we reach that")
test.That(t, z.Pos() == 0, "buffer must start at position 0")
test.That(t, z.Peek(0) == 'L', "first character must be 'L'")
test.That(t, z.Peek(1) == 'o', "second character must be 'o'")
z.Move(1)
test.That(t, z.Peek(0) == 'o', "must be 'o' at position 1")
test.That(t, z.Peek(1) == 'r', "must be 'r' at position 2")
z.Rewind(6)
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 6")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 7")
test.T(t, z.Offset(), 6, "offset")
test.Bytes(t, z.Lexeme(), []byte("Lorem "), "buffered string must now read 'Lorem ' when at position 6")
test.Bytes(t, z.Shift(), []byte("Lorem "), "shift must return the buffered string")
test.That(t, z.Pos() == 0, "after shifting position must be 0")
test.That(t, z.Peek(0) == 'i', "must be 'i' at position 0 after shifting")
test.That(t, z.Peek(1) == 'p', "must be 'p' at position 1 after shifting")
test.T(t, z.Err(), nil, "error must be nil at this point")
z.Move(len(s) - len("Lorem ") - 1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer")
z.Skip()
test.That(t, z.Pos() == 0, "after skipping position must be 0")
z.Move(1)
test.T(t, z.Err(), io.EOF, "error must be EOF when past the buffer")
z.Move(-1)
test.T(t, z.Err(), nil, "error must be nil just before the end of the buffer, even when it has been past the buffer")
z.Reset()
test.That(t, z.Peek(0) == 'L', "must be 'L' at position 0")
test.That(t, z.Peek(1) == 'o', "must be 'o' at position 1")
test.T(t, z.Len(), len(s))
}
func TestInputRunes(t *testing.T) {
z := NewInput(bytes.NewBufferString("aæ†\U00100000"))
r, n := z.PeekRune(0)
test.That(t, n == 1, "first character must be length 1")
test.That(t, r == 'a', "first character must be rune 'a'")
r, n = z.PeekRune(1)
test.That(t, n == 2, "second character must be length 2")
test.That(t, r == 'æ', "second character must be rune 'æ'")
r, n = z.PeekRune(3)
test.That(t, n == 3, "fourth character must be length 3")
test.That(t, r == '†', "fourth character must be rune '†'")
r, n = z.PeekRune(6)
test.That(t, n == 4, "seventh character must be length 4")
test.That(t, r == '\U00100000', "seventh character must be rune '\U00100000'")
}
func TestInputBadRune(t *testing.T) {
z := NewInput(bytes.NewBufferString("\xF0")) // expect four byte rune
r, n := z.PeekRune(0)
test.T(t, n, 1, "length")
test.T(t, r, rune(0xF0), "rune")
}
func TestInputZeroLen(t *testing.T) {
z := NewInput(test.NewPlainReader(bytes.NewBufferString("")))
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.Bytes(t, z.Bytes(), []byte{}, "bytes match original buffer")
}
func TestInputEmptyInput(t *testing.T) {
z := NewInput(test.NewEmptyReader())
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), io.EOF, "error must be EOF")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
func TestInputErrorInput(t *testing.T) {
z := NewInput(test.NewErrorReader(0))
test.That(t, z.Peek(0) == 0, "first character must yield error")
test.T(t, z.Err(), test.ErrPlain, "error must be ErrPlain")
test.That(t, z.Peek(0) == 0, "second peek must also yield error")
}
func TestInputBytes(t *testing.T) {
b := []byte{'t', 'e', 's', 't'}
z := NewInputBytes(b)
test.That(t, z.Peek(4) == 0, "fifth character must yield NULL")
}
func TestInputRestore(t *testing.T) {
b := []byte{'a', 'b', 'c', 'd'}
z := NewInputBytes(b[:2])
test.T(t, len(z.buf), 3, "must have terminating NULL")
test.T(t, z.buf[2], byte(0), "must have terminating NULL")
test.Bytes(t, b, []byte{'a', 'b', 0, 'd'}, "terminating NULL overwrites underlying buffer")
z.Restore()
test.Bytes(t, b, []byte{'a', 'b', 'c', 'd'}, "terminating NULL has been restored")
}
parse-2.5.21/js/ 0000775 0000000 0000000 00000000000 14117006534 0013330 5 ustar 00root root 0000000 0000000 parse-2.5.21/js/README.md 0000664 0000000 0000000 00000004230 14117006534 0014606 0 ustar 00root root 0000000 0000000 # JS [](https://pkg.go.dev/github.com/tdewolff/minify/v2/parse/js?tab=doc)
This package is a JS lexer (ECMAScript 2020) written in [Go][1]. It follows the specification at [ECMAScript 2020 Language Specification](https://tc39.es/ecma262/). The lexer takes an io.Reader and converts it into tokens until the EOF.
## Installation
Run the following command
go get -u github.com/tdewolff/parse/v2/js
or add the following import and run project with `go get`
import "github.com/tdewolff/parse/v2/js"
## Lexer
### Usage
The following initializes a new Lexer with io.Reader `r`:
``` go
l := js.NewLexer(parse.NewInput(r))
```
To tokenize until EOF an error, use:
``` go
for {
tt, text := l.Next()
switch tt {
case js.ErrorToken:
// error or EOF set in l.Err()
return
// ...
}
}
```
### Regular Expressions
The ECMAScript specification for `PunctuatorToken` (of which the `/` and `/=` symbols) and `RegExpToken` depend on a parser state to differentiate between the two. The lexer will always parse the first token as `/` or `/=` operator, upon which the parser can rescan that token to scan a regular expression using `RegExp()`.
### Examples
``` go
package main
import (
"os"
"github.com/tdewolff/parse/v2/js"
)
// Tokenize JS from stdin.
func main() {
l := js.NewLexer(parse.NewInput(os.Stdin))
for {
tt, text := l.Next()
switch tt {
case js.ErrorToken:
if l.Err() != io.EOF {
fmt.Println("Error on line", l.Line(), ":", l.Err())
}
return
case js.IdentifierToken:
fmt.Println("Identifier", string(text))
case js.NumericToken:
fmt.Println("Numeric", string(text))
// ...
}
}
}
```
## Parser
### Usage
The following parses a file and returns an abstract syntax tree (AST).
``` go
ast, err := js.NewParser(parse.NewInputString("if (state == 5) { console.log('In state five'); }"))
```
See [ast.go](https://github.com/tdewolff/parse/blob/master/js/ast.go) for all available data structures that can represent the abstact syntax tree.
## License
Released under the [MIT license](https://github.com/tdewolff/parse/blob/master/LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/js/ast.go 0000664 0000000 0000000 00000143065 14117006534 0014457 0 ustar 00root root 0000000 0000000 package js
import (
"bytes"
"fmt"
"strconv"
)
var ErrInvalidJSON = fmt.Errorf("invalid JSON")
// AST is the full ECMAScript abstract syntax tree.
type AST struct {
Comments [][]byte // first comments in file
BlockStmt // module
}
func (ast *AST) String() string {
s := ""
for i, item := range ast.BlockStmt.List {
if i != 0 {
s += " "
}
s += item.String()
}
return s
}
////////////////////////////////////////////////////////////////
// DeclType specifies the kind of declaration.
type DeclType uint16
// DeclType values.
const (
NoDecl DeclType = iota // undeclared variables
VariableDecl // var
FunctionDecl // function
ArgumentDecl // function and method arguments
LexicalDecl // let, const, class
CatchDecl // catch statement argument
ExprDecl // function expression name or class expression name
)
func (decl DeclType) String() string {
switch decl {
case NoDecl:
return "NoDecl"
case VariableDecl:
return "VariableDecl"
case FunctionDecl:
return "FunctionDecl"
case ArgumentDecl:
return "ArgumentDecl"
case LexicalDecl:
return "LexicalDecl"
case CatchDecl:
return "CatchDecl"
case ExprDecl:
return "ExprDecl"
}
return "Invalid(" + strconv.Itoa(int(decl)) + ")"
}
// Var is a variable, where Decl is the type of declaration and can be var|function for function scoped variables, let|const|class for block scoped variables.
type Var struct {
Data []byte
Link *Var // is set when merging variable uses, as in: {a} {var a} where the first links to the second, only used for undeclared variables
Uses uint16
Decl DeclType
}
// Name returns the variable name.
func (v *Var) Name() []byte {
for v.Link != nil {
v = v.Link
}
return v.Data
}
func (v Var) String() string {
return string(v.Name())
}
// JS converts the node back to valid JavaScript
func (v Var) JS() string {
return v.String()
}
// JSON converts the node back to valid JSON
func (n Var) JSON() (string, error) {
return "", ErrInvalidJSON
}
// VarsByUses is sortable by uses in descending order.
type VarsByUses VarArray
func (vs VarsByUses) Len() int {
return len(vs)
}
func (vs VarsByUses) Swap(i, j int) {
vs[i], vs[j] = vs[j], vs[i]
}
func (vs VarsByUses) Less(i, j int) bool {
return vs[i].Uses > vs[j].Uses
}
////////////////////////////////////////////////////////////////
// VarArray is a set of variables in scopes.
type VarArray []*Var
func (vs VarArray) String() string {
s := "["
for i, v := range vs {
if i != 0 {
s += ", "
}
links := 0
for v.Link != nil {
v = v.Link
links++
}
s += fmt.Sprintf("Var{%v %s %v %v}", v.Decl, string(v.Data), links, v.Uses)
}
return s + "]"
}
// Scope is a function or block scope with a list of variables declared and used.
type Scope struct {
Parent, Func *Scope // Parent is nil for global scope, Parent equals Func for function scope
Declared VarArray // Link in Var are always nil
Undeclared VarArray
NumVarDecls uint16 // number of variable declaration statements in a function scope
NumForInit uint16 // offset into Declared to mark variables used in for initializer
NumArguments uint16 // offset into Undeclared to mark variables used in arguments
IsGlobalOrFunc bool
HasWith bool
}
func (s Scope) String() string {
return "Scope{Declared: " + s.Declared.String() + ", Undeclared: " + s.Undeclared.String() + "}"
}
// Declare declares a new variable.
func (s *Scope) Declare(decl DeclType, name []byte) (*Var, bool) {
// refer to new variable for previously undeclared symbols in the current and lower scopes
// this happens in `{ a = 5; } var a` where both a's refer to the same variable
curScope := s
if decl == VariableDecl || decl == FunctionDecl {
// find function scope for var and function declarations
for s != s.Func {
// make sure that `{let i;{var i}}` is an error
if v := s.findDeclared(name, false); v != nil && v.Decl != decl && v.Decl != CatchDecl {
return nil, false
}
s = s.Parent
}
}
if v := s.findDeclared(name, true); v != nil {
// variable already declared, might be an error or a duplicate declaration
if (LexicalDecl <= v.Decl || LexicalDecl <= decl) && v.Decl != ExprDecl {
// redeclaration of let, const, class on an already declared name is an error, except if the declared name is a function expression name
return nil, false
}
if v.Decl == ExprDecl {
v.Decl = decl
}
v.Uses++
for s != curScope {
curScope.addUndeclared(v) // add variable declaration as used variable to the current scope
curScope = curScope.Parent
}
return v, true
}
var v *Var
// reuse variable if previously used, as in: a;var a
if decl != ArgumentDecl { // in case of function f(a=b,b), where the first b is different from the second
for i, uv := range s.Undeclared[s.NumArguments:] {
// no need to evaluate v.Link as v.Data stays the same and Link is nil in the active scope
if 0 < uv.Uses && uv.Decl == NoDecl && bytes.Equal(name, uv.Data) {
// must be NoDecl so that it can't be a var declaration that has been added
v = uv
s.Undeclared = append(s.Undeclared[:int(s.NumArguments)+i], s.Undeclared[int(s.NumArguments)+i+1:]...)
break
}
}
}
if v == nil {
// add variable to the context list and to the scope
v = &Var{name, nil, 0, decl}
} else {
v.Decl = decl
}
v.Uses++
s.Declared = append(s.Declared, v)
for s != curScope {
curScope.addUndeclared(v) // add variable declaration as used variable to the current scope
curScope = curScope.Parent
}
return v, true
}
// Use increments the usage of a variable.
func (s *Scope) Use(name []byte) *Var {
// check if variable is declared in the current scope
v := s.findDeclared(name, false)
if v == nil {
// check if variable is already used before in the current or lower scopes
v = s.findUndeclared(name)
if v == nil {
// add variable to the context list and to the scope's undeclared
v = &Var{name, nil, 0, NoDecl}
s.Undeclared = append(s.Undeclared, v)
}
}
v.Uses++
return v
}
// findDeclared finds a declared variable in the current scope.
func (s *Scope) findDeclared(name []byte, skipForInit bool) *Var {
start := 0
if skipForInit {
// we skip the for initializer for declarations (only has effect for let/const)
start = int(s.NumForInit)
}
// reverse order to find the inner let first in `for(let a in []){let a; {a}}`
for i := len(s.Declared) - 1; start <= i; i-- {
v := s.Declared[i]
// no need to evaluate v.Link as v.Data stays the same, and Link is always nil in Declared
if bytes.Equal(name, v.Data) {
return v
}
}
return nil
}
// findUndeclared finds an undeclared variable in the current and contained scopes.
func (s *Scope) findUndeclared(name []byte) *Var {
for _, v := range s.Undeclared {
// no need to evaluate v.Link as v.Data stays the same and Link is nil in the active scope
if 0 < v.Uses && bytes.Equal(name, v.Data) {
return v
}
}
return nil
}
// add undeclared variable to scope, this is called for the block scope when declaring a var in it
func (s *Scope) addUndeclared(v *Var) {
// don't add undeclared symbol if it's already there
for _, vorig := range s.Undeclared {
if v == vorig {
return
}
}
s.Undeclared = append(s.Undeclared, v) // add variable declaration as used variable to the current scope
}
// MarkForInit marks the declared variables in current scope as for statement initializer to distinguish from declarations in body.
func (s *Scope) MarkForInit() {
s.NumForInit = uint16(len(s.Declared))
}
// MarkArguments marks the undeclared variables in the current scope as function arguments. It ensures different b's in `function f(a=b){var b}`.
func (s *Scope) MarkArguments() {
s.NumArguments = uint16(len(s.Undeclared))
}
// HoistUndeclared copies all undeclared variables of the current scope to the parent scope.
func (s *Scope) HoistUndeclared() {
for i, vorig := range s.Undeclared {
// no need to evaluate vorig.Link as vorig.Data stays the same
if 0 < vorig.Uses && vorig.Decl == NoDecl {
if v := s.Parent.findDeclared(vorig.Data, false); v != nil {
// check if variable is declared in parent scope
v.Uses += vorig.Uses
vorig.Link = v
s.Undeclared[i] = v // point reference to existing var (to avoid many Link chains)
} else if v := s.Parent.findUndeclared(vorig.Data); v != nil {
// check if variable is already used before in parent scope
v.Uses += vorig.Uses
vorig.Link = v
s.Undeclared[i] = v // point reference to existing var (to avoid many Link chains)
} else {
// add variable to the context list and to the scope's undeclared
s.Parent.Undeclared = append(s.Parent.Undeclared, vorig)
}
}
}
}
// UndeclareScope undeclares all declared variables in the current scope and adds them to the parent scope.
// Called when possible arrow func ends up being a parenthesized expression, scope is not further used.
func (s *Scope) UndeclareScope() {
// look if the variable already exists in the parent scope, if so replace the Var pointer in original use
for _, vorig := range s.Declared {
// no need to evaluate vorig.Link as vorig.Data stays the same, and Link is always nil in Declared
// vorig.Uses will be atleast 1
if v := s.Parent.findDeclared(vorig.Data, false); v != nil {
// check if variable has been declared in this scope
v.Uses += vorig.Uses
vorig.Link = v
} else if v := s.Parent.findUndeclared(vorig.Data); v != nil {
// check if variable is already used before in the current or lower scopes
v.Uses += vorig.Uses
vorig.Link = v
} else {
// add variable to the context list and to the scope's undeclared
vorig.Decl = NoDecl
s.Parent.Undeclared = append(s.Parent.Undeclared, vorig)
}
}
s.Declared = s.Declared[:0]
s.Undeclared = s.Undeclared[:0]
}
// Unscope moves all declared variables of the current scope to the parent scope. Undeclared variables are already in the parent scope.
func (s *Scope) Unscope() {
for _, vorig := range s.Declared {
// no need to evaluate vorig.Link as vorig.Data stays the same, and Link is always nil in Declared
// vorig.Uses will be atleast 1
s.Parent.Declared = append(s.Parent.Declared, vorig)
}
s.Declared = s.Declared[:0]
s.Undeclared = s.Undeclared[:0]
}
////////////////////////////////////////////////////////////////
// INode is an interface for AST nodes
type INode interface {
String() string
JS() string
JSON() (string, error)
}
// IStmt is a dummy interface for statements.
type IStmt interface {
INode
stmtNode()
}
// IBinding is a dummy interface for bindings.
type IBinding interface {
INode
bindingNode()
}
// IExpr is a dummy interface for expressions.
type IExpr interface {
INode
exprNode()
}
////////////////////////////////////////////////////////////////
// BlockStmt is a block statement.
type BlockStmt struct {
List []IStmt
Scope
}
func (n BlockStmt) String() string {
s := "Stmt({"
for _, item := range n.List {
s += " " + item.String()
}
return s + " })"
}
// JS converts the node back to valid JavaScript
func (n BlockStmt) JS() string {
s := ""
if n.Scope.Parent != nil {
s += "{ "
}
for _, item := range n.List {
s += item.JS() + "; "
}
if n.Scope.Parent != nil {
s += "}"
}
return s
}
// JSON converts the node back to valid JSON
func (n BlockStmt) JSON() (string, error) {
if len(n.List) != 1 {
return "", ErrInvalidJSON
}
return n.List[0].JSON()
}
// EmptyStmt is an empty statement.
type EmptyStmt struct {
}
func (n EmptyStmt) String() string {
return "Stmt(;)"
}
// JS converts the node back to valid JavaScript
func (n EmptyStmt) JS() string {
return ";"
}
// JSON converts the node back to valid JSON
func (n EmptyStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ExprStmt is an expression statement.
type ExprStmt struct {
Value IExpr
}
func (n ExprStmt) String() string {
val := n.Value.String()
if val[0] == '(' && val[len(val)-1] == ')' {
return "Stmt" + n.Value.String()
}
return "Stmt(" + n.Value.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ExprStmt) JS() string {
return n.Value.JS()
}
// JSON converts the node back to valid JSON
func (n ExprStmt) JSON() (string, error) {
return n.Value.JSON()
}
// IfStmt is an if statement.
type IfStmt struct {
Cond IExpr
Body IStmt
Else IStmt // can be nil
}
func (n IfStmt) String() string {
s := "Stmt(if " + n.Cond.String() + " " + n.Body.String()
if n.Else != nil {
s += " else " + n.Else.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n IfStmt) JS() string {
s := "if (" + n.Cond.JS() + ") "
switch n.Body.(type) {
case *BlockStmt:
s += n.Body.JS()
default:
s += "{ " + n.Body.JS() + " }"
}
if n.Else != nil {
switch n.Else.(type) {
case *BlockStmt:
s += " else " + n.Else.JS()
default:
s += " else { " + n.Else.JS() + " }"
}
}
return s
}
// JSON converts the node back to valid JSON
func (n IfStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// DoWhileStmt is a do-while iteration statement.
type DoWhileStmt struct {
Cond IExpr
Body IStmt
}
func (n DoWhileStmt) String() string {
return "Stmt(do " + n.Body.String() + " while " + n.Cond.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n DoWhileStmt) JS() string {
s := "do "
switch n.Body.(type) {
case *BlockStmt:
s += n.Body.JS()
default:
s += "{ " + n.Body.JS() + " }"
}
return s + " while (" + n.Cond.JS() + ")"
}
// JSON converts the node back to valid JSON
func (n DoWhileStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// WhileStmt is a while iteration statement.
type WhileStmt struct {
Cond IExpr
Body IStmt
}
func (n WhileStmt) String() string {
return "Stmt(while " + n.Cond.String() + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n WhileStmt) JS() string {
s := "while (" + n.Cond.JS() + ") "
if n.Body != nil {
s += n.Body.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n WhileStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ForStmt is a regular for iteration statement.
type ForStmt struct {
Init IExpr // can be nil
Cond IExpr // can be nil
Post IExpr // can be nil
Body *BlockStmt
}
func (n ForStmt) String() string {
s := "Stmt(for"
if n.Init != nil {
s += " " + n.Init.String()
}
s += " ;"
if n.Cond != nil {
s += " " + n.Cond.String()
}
s += " ;"
if n.Post != nil {
s += " " + n.Post.String()
}
return s + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ForStmt) JS() string {
s := "for ("
if n.Init != nil {
s += n.Init.JS()
} else {
s += " "
}
s += "; "
if n.Cond != nil {
s += n.Cond.JS()
}
s += "; "
if n.Post != nil {
s += n.Post.JS()
}
return s + ") " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n ForStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ForInStmt is a for-in iteration statement.
type ForInStmt struct {
Init IExpr
Value IExpr
Body *BlockStmt
}
func (n ForInStmt) String() string {
return "Stmt(for " + n.Init.String() + " in " + n.Value.String() + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ForInStmt) JS() string {
return "for (" + n.Init.JS() + " in " + n.Value.JS() + ") " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n ForInStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ForOfStmt is a for-of iteration statement.
type ForOfStmt struct {
Await bool
Init IExpr
Value IExpr
Body *BlockStmt
}
func (n ForOfStmt) String() string {
s := "Stmt(for"
if n.Await {
s += " await"
}
return s + " " + n.Init.String() + " of " + n.Value.String() + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ForOfStmt) JS() string {
s := "for"
if n.Await {
s += " await"
}
return s + " (" + n.Init.JS() + " of " + n.Value.JS() + ") " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n ForOfStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// CaseClause is a case clause or default clause for a switch statement.
type CaseClause struct {
TokenType
Cond IExpr // can be nil
List []IStmt
}
func (n CaseClause) String() string {
s := " Clause(" + n.TokenType.String()
if n.Cond != nil {
s += " " + n.Cond.String()
}
for _, item := range n.List {
s += " " + item.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n CaseClause) JS() string {
s := " "
if n.Cond != nil {
s += "case " + n.Cond.JS()
} else {
s += "default"
}
s += ":"
for _, item := range n.List {
s += " " + item.JS() + ";"
}
return s
}
// JSON converts the node back to valid JSON
func (n CaseClause) JSON() (string, error) {
return "", ErrInvalidJSON
}
// SwitchStmt is a switch statement.
type SwitchStmt struct {
Init IExpr
List []CaseClause
Scope
}
func (n SwitchStmt) String() string {
s := "Stmt(switch " + n.Init.String()
for _, clause := range n.List {
s += clause.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n SwitchStmt) JS() string {
s := "switch (" + n.Init.JS() + ") {"
for _, clause := range n.List {
s += clause.JS()
}
return s + " }"
}
// JSON converts the node back to valid JSON
func (n SwitchStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BranchStmt is a continue or break statement.
type BranchStmt struct {
Type TokenType
Label []byte // can be nil
}
func (n BranchStmt) String() string {
s := "Stmt(" + n.Type.String()
if n.Label != nil {
s += " " + string(n.Label)
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n BranchStmt) JS() string {
s := n.Type.String()
if n.Label != nil {
s += " " + string(n.Label)
}
return s
}
// JSON converts the node back to valid JSON
func (n BranchStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ReturnStmt is a return statement.
type ReturnStmt struct {
Value IExpr // can be nil
}
func (n ReturnStmt) String() string {
s := "Stmt(return"
if n.Value != nil {
s += " " + n.Value.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n ReturnStmt) JS() string {
s := "return"
if n.Value != nil {
s += " " + n.Value.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n ReturnStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// WithStmt is a with statement.
type WithStmt struct {
Cond IExpr
Body IStmt
}
func (n WithStmt) String() string {
return "Stmt(with " + n.Cond.String() + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n WithStmt) JS() string {
return "with (" + n.Cond.JS() + ") " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n WithStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// LabelledStmt is a labelled statement.
type LabelledStmt struct {
Label []byte
Value IStmt
}
func (n LabelledStmt) String() string {
return "Stmt(" + string(n.Label) + " : " + n.Value.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n LabelledStmt) JS() string {
return string(n.Label) + ": " + n.Value.JS()
}
// JSON converts the node back to valid JSON
func (n LabelledStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ThrowStmt is a throw statement.
type ThrowStmt struct {
Value IExpr
}
func (n ThrowStmt) String() string {
return "Stmt(throw " + n.Value.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ThrowStmt) JS() string {
return "throw " + n.Value.JS()
}
// JSON converts the node back to valid JSON
func (n ThrowStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// TryStmt is a try statement.
type TryStmt struct {
Body *BlockStmt
Binding IBinding // can be nil
Catch *BlockStmt // can be nil
Finally *BlockStmt // can be nil
}
func (n TryStmt) String() string {
s := "Stmt(try " + n.Body.String()
if n.Catch != nil {
s += " catch"
if n.Binding != nil {
s += " Binding(" + n.Binding.String() + ")"
}
s += " " + n.Catch.String()
}
if n.Finally != nil {
s += " finally " + n.Finally.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n TryStmt) JS() string {
s := "try " + n.Body.JS()
if n.Catch != nil {
s += " catch"
if n.Binding != nil {
s += "(" + n.Binding.JS() + ")"
}
s += " " + n.Catch.JS()
}
if n.Finally != nil {
s += " finally " + n.Finally.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n TryStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// DebuggerStmt is a debugger statement.
type DebuggerStmt struct {
}
func (n DebuggerStmt) String() string {
return "Stmt(debugger)"
}
// JS converts the node back to valid JavaScript
func (n DebuggerStmt) JS() string {
return "debugger"
}
// JSON converts the node back to valid JSON
func (n DebuggerStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// Alias is a name space import or import/export specifier for import/export statements.
type Alias struct {
Name []byte // can be nil
Binding []byte // can be nil
}
func (alias Alias) String() string {
s := ""
if alias.Name != nil {
s += string(alias.Name) + " as "
}
return s + string(alias.Binding)
}
// JS converts the node back to valid JavaScript
func (alias Alias) JS() string {
return alias.String()
}
// JSON converts the node back to valid JSON
func (n Alias) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ImportStmt is an import statement.
type ImportStmt struct {
List []Alias
Default []byte // can be nil
Module []byte
}
func (n ImportStmt) String() string {
s := "Stmt(import"
if n.Default != nil {
s += " " + string(n.Default)
if len(n.List) != 0 {
s += " ,"
}
}
if len(n.List) == 1 && len(n.List[0].Name) == 1 && n.List[0].Name[0] == '*' {
s += " " + n.List[0].String()
} else if 0 < len(n.List) {
s += " {"
for i, item := range n.List {
if i != 0 {
s += " ,"
}
if item.Binding != nil {
s += " " + item.String()
}
}
s += " }"
}
if n.Default != nil || len(n.List) != 0 {
s += " from"
}
return s + " " + string(n.Module) + ")"
}
// JS converts the node back to valid JavaScript
func (n ImportStmt) JS() string {
s := "import"
if n.Default != nil {
s += " " + string(n.Default)
if len(n.List) != 0 {
s += " ,"
}
}
if len(n.List) == 1 && len(n.List[0].Name) == 1 && n.List[0].Name[0] == '*' {
s += " " + n.List[0].JS()
} else if 0 < len(n.List) {
s += " {"
for i, item := range n.List {
if i != 0 {
s += " ,"
}
if item.Binding != nil {
s += " " + item.JS()
}
}
s += " }"
}
if n.Default != nil || len(n.List) != 0 {
s += " from"
}
return s + " " + string(n.Module)
}
// JSON converts the node back to valid JSON
func (n ImportStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ExportStmt is an export statement.
type ExportStmt struct {
List []Alias
Module []byte // can be nil
Default bool
Decl IExpr
}
func (n ExportStmt) String() string {
s := "Stmt(export"
if n.Decl != nil {
if n.Default {
s += " default"
}
return s + " " + n.Decl.String() + ")"
} else if len(n.List) == 1 && (len(n.List[0].Name) == 1 && n.List[0].Name[0] == '*' || n.List[0].Name == nil && len(n.List[0].Binding) == 1 && n.List[0].Binding[0] == '*') {
s += " " + n.List[0].String()
} else if 0 < len(n.List) {
s += " {"
for i, item := range n.List {
if i != 0 {
s += " ,"
}
if item.Binding != nil {
s += " " + item.String()
}
}
s += " }"
}
if n.Module != nil {
s += " from " + string(n.Module)
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n ExportStmt) JS() string {
s := "export"
if n.Decl != nil {
if n.Default {
s += " default"
}
return s + " " + n.Decl.JS()
} else if len(n.List) == 1 && (len(n.List[0].Name) == 1 && n.List[0].Name[0] == '*' || n.List[0].Name == nil && len(n.List[0].Binding) == 1 && n.List[0].Binding[0] == '*') {
s += " " + n.List[0].JS()
} else if 0 < len(n.List) {
s += " {"
for i, item := range n.List {
if i != 0 {
s += " ,"
}
if item.Binding != nil {
s += " " + item.JS()
}
}
s += " }"
}
if n.Module != nil {
s += " from " + string(n.Module)
}
return s
}
// JSON converts the node back to valid JSON
func (n ExportStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
// DirectivePrologueStmt is a string literal at the beginning of a function or module (usually "use strict").
type DirectivePrologueStmt struct {
Value []byte
}
func (n DirectivePrologueStmt) String() string {
return "Stmt(" + string(n.Value) + ")"
}
// JS converts the node back to valid JavaScript
func (n DirectivePrologueStmt) JS() string {
return string(n.Value)
}
// JSON converts the node back to valid JSON
func (n DirectivePrologueStmt) JSON() (string, error) {
return "", ErrInvalidJSON
}
func (n BlockStmt) stmtNode() {}
func (n EmptyStmt) stmtNode() {}
func (n ExprStmt) stmtNode() {}
func (n IfStmt) stmtNode() {}
func (n DoWhileStmt) stmtNode() {}
func (n WhileStmt) stmtNode() {}
func (n ForStmt) stmtNode() {}
func (n ForInStmt) stmtNode() {}
func (n ForOfStmt) stmtNode() {}
func (n SwitchStmt) stmtNode() {}
func (n BranchStmt) stmtNode() {}
func (n ReturnStmt) stmtNode() {}
func (n WithStmt) stmtNode() {}
func (n LabelledStmt) stmtNode() {}
func (n ThrowStmt) stmtNode() {}
func (n TryStmt) stmtNode() {}
func (n DebuggerStmt) stmtNode() {}
func (n ImportStmt) stmtNode() {}
func (n ExportStmt) stmtNode() {}
func (n DirectivePrologueStmt) stmtNode() {}
////////////////////////////////////////////////////////////////
// PropertyName is a property name for binding properties, method names, and in object literals.
type PropertyName struct {
Literal LiteralExpr
Computed IExpr // can be nil
}
// IsSet returns true is PropertyName is not nil.
func (n PropertyName) IsSet() bool {
return n.IsComputed() || n.Literal.TokenType != ErrorToken
}
// IsComputed returns true if PropertyName is computed.
func (n PropertyName) IsComputed() bool {
return n.Computed != nil
}
// IsIdent returns true if PropertyName equals the given identifier name.
func (n PropertyName) IsIdent(data []byte) bool {
return !n.IsComputed() && n.Literal.TokenType == IdentifierToken && bytes.Equal(data, n.Literal.Data)
}
func (n PropertyName) String() string {
if n.Computed != nil {
val := n.Computed.String()
if val[0] == '(' {
return "[" + val[1:len(val)-1] + "]"
}
return "[" + val + "]"
}
return string(n.Literal.Data)
}
// JS converts the node back to valid JavaScript
func (n PropertyName) JS() string {
return n.String()
}
// JSON converts the node back to valid JSON
func (n PropertyName) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BindingArray is an array binding pattern.
type BindingArray struct {
List []BindingElement
Rest IBinding // can be nil
}
func (n BindingArray) String() string {
s := "["
for i, item := range n.List {
if i != 0 {
s += ","
}
s += " " + item.String()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ","
}
s += " ...Binding(" + n.Rest.String() + ")"
}
return s + " ]"
}
// JS converts the node back to valid JavaScript
func (n BindingArray) JS() string {
s := "["
for i, item := range n.List {
if i != 0 {
s += ","
}
s += item.JS()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ","
}
s += " ..." + n.Rest.JS()
}
return s + "]"
}
// JSON converts the node back to valid JSON
func (n BindingArray) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BindingObjectItem is a binding property.
type BindingObjectItem struct {
Key *PropertyName // can be nil
Value BindingElement
}
func (n BindingObjectItem) String() string {
s := ""
if n.Key != nil {
if v, ok := n.Value.Binding.(*Var); !ok || !n.Key.IsIdent(v.Data) {
s += " " + n.Key.String() + ":"
}
}
return " " + n.Value.String()
}
// JS converts the node back to valid JavaScript
func (n BindingObjectItem) JS() string {
s := ""
if n.Key != nil {
if v, ok := n.Value.Binding.(*Var); !ok || !n.Key.IsIdent(v.Data) {
s += " " + n.Key.JS() + ":"
}
}
return " " + n.Value.JS()
}
// JSON converts the node back to valid JSON
func (n BindingObjectItem) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BindingObject is an object binding pattern.
type BindingObject struct {
List []BindingObjectItem
Rest *Var // can be nil
}
func (n BindingObject) String() string {
s := "{"
for i, item := range n.List {
if i != 0 {
s += ","
}
if item.Key != nil {
if v, ok := item.Value.Binding.(*Var); !ok || !item.Key.IsIdent(v.Data) {
s += " " + item.Key.String() + ":"
}
}
s += " " + item.Value.String()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ","
}
s += " ...Binding(" + string(n.Rest.Data) + ")"
}
return s + " }"
}
// JS converts the node back to valid JavaScript
func (n BindingObject) JS() string {
s := "{"
for i, item := range n.List {
if i != 0 {
s += ","
}
if item.Key != nil {
if v, ok := item.Value.Binding.(*Var); !ok || !item.Key.IsIdent(v.Data) {
s += " " + item.Key.JS() + ":"
}
}
s += " " + item.Value.JS()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ","
}
s += " ..." + string(n.Rest.Data)
}
return s + " }"
}
// JSON converts the node back to valid JSON
func (n BindingObject) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BindingElement is a binding element.
type BindingElement struct {
Binding IBinding // can be nil (in case of ellision)
Default IExpr // can be nil
}
func (n BindingElement) String() string {
if n.Binding == nil {
return "Binding()"
}
s := "Binding(" + n.Binding.String()
if n.Default != nil {
s += " = " + n.Default.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n BindingElement) JS() string {
if n.Binding == nil {
return ""
}
s := n.Binding.JS()
if n.Default != nil {
s += " = " + n.Default.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n BindingElement) JSON() (string, error) {
return "", ErrInvalidJSON
}
func (v *Var) bindingNode() {}
func (n BindingArray) bindingNode() {}
func (n BindingObject) bindingNode() {}
////////////////////////////////////////////////////////////////
// VarDecl is a variable statement or lexical declaration.
type VarDecl struct {
TokenType
List []BindingElement
}
func (n VarDecl) String() string {
s := "Decl(" + n.TokenType.String()
for _, item := range n.List {
s += " " + item.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n VarDecl) JS() string {
s := n.TokenType.String()
for i, item := range n.List {
if i != 0 {
s += ","
}
s += " " + item.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n VarDecl) JSON() (string, error) {
return "", ErrInvalidJSON
}
// Params is a list of parameters for functions, methods, and arrow function.
type Params struct {
List []BindingElement
Rest IBinding // can be nil
}
func (n Params) String() string {
s := "Params("
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.String()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ", "
}
s += "...Binding(" + n.Rest.String() + ")"
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n Params) JS() string {
s := "("
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.JS()
}
if n.Rest != nil {
if len(n.List) != 0 {
s += ", "
}
s += "..." + n.Rest.JS()
}
return s + ")"
}
// JSON converts the node back to valid JSON
func (n Params) JSON() (string, error) {
return "", ErrInvalidJSON
}
// FuncDecl is an (async) (generator) function declaration or expression.
type FuncDecl struct {
Async bool
Generator bool
Name *Var // can be nil
Params Params
Body BlockStmt
}
func (n FuncDecl) String() string {
s := "Decl("
if n.Async {
s += "async function"
} else {
s += "function"
}
if n.Generator {
s += "*"
}
if n.Name != nil {
s += " " + string(n.Name.Data)
}
return s + " " + n.Params.String() + " " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n FuncDecl) JS() string {
s := ""
if n.Async {
s += "async function"
} else {
s += "function"
}
if n.Generator {
s += "*"
}
if n.Name != nil {
s += " " + string(n.Name.Data)
}
return s + " " + n.Params.JS() + " " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n FuncDecl) JSON() (string, error) {
return "", ErrInvalidJSON
}
// MethodDecl is a method definition in a class declaration.
type MethodDecl struct {
Static bool
Async bool
Generator bool
Get bool
Set bool
Name PropertyName
Params Params
Body BlockStmt
}
func (n MethodDecl) String() string {
s := ""
if n.Static {
s += " static"
}
if n.Async {
s += " async"
}
if n.Generator {
s += " *"
}
if n.Get {
s += " get"
}
if n.Set {
s += " set"
}
s += " " + n.Name.String() + " " + n.Params.String() + " " + n.Body.String()
return "Method(" + s[1:] + ")"
}
// JS converts the node back to valid JavaScript
func (n MethodDecl) JS() string {
s := ""
if n.Static {
s += " static"
}
if n.Async {
s += " async"
}
if n.Generator {
s += " *"
}
if n.Get {
s += " get"
}
if n.Set {
s += " set"
}
s += " " + n.Name.JS() + " " + n.Params.JS() + " " + n.Body.JS()
return s[1:]
}
// JSON converts the node back to valid JSON
func (n MethodDecl) JSON() (string, error) {
return "", ErrInvalidJSON
}
// FieldDefinition is a field definition in a class declaration.
type FieldDefinition struct {
Name PropertyName
Init IExpr
}
func (n FieldDefinition) String() string {
s := "Definition(" + n.Name.String()
if n.Init != nil {
s += " = " + n.Init.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n FieldDefinition) JS() string {
s := n.Name.String()
if n.Init != nil {
s += " = " + n.Init.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n FieldDefinition) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ClassDecl is a class declaration.
type ClassDecl struct {
Name *Var // can be nil
Extends IExpr // can be nil
Definitions []FieldDefinition
Methods []*MethodDecl
}
func (n ClassDecl) String() string {
s := "Decl(class"
if n.Name != nil {
s += " " + string(n.Name.Data)
}
if n.Extends != nil {
s += " extends " + n.Extends.String()
}
for _, item := range n.Definitions {
s += " " + item.String()
}
for _, item := range n.Methods {
s += " " + item.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n ClassDecl) JS() string {
s := "class"
if n.Name != nil {
s += " " + string(n.Name.Data)
}
if n.Extends != nil {
s += " extends " + n.Extends.JS()
}
s += " { "
for _, item := range n.Definitions {
s += item.JS() + "; "
}
for _, item := range n.Methods {
s += item.JS() + "; "
}
return s + "}"
}
// JSON converts the node back to valid JSON
func (n ClassDecl) JSON() (string, error) {
return "", ErrInvalidJSON
}
func (n VarDecl) stmtNode() {}
func (n FuncDecl) stmtNode() {}
func (n ClassDecl) stmtNode() {}
func (n VarDecl) exprNode() {} // not a real IExpr, used for ForInit and ExportDecl
func (n FuncDecl) exprNode() {}
func (n ClassDecl) exprNode() {}
func (n MethodDecl) exprNode() {} // not a real IExpr, used for ObjectExpression PropertyName
////////////////////////////////////////////////////////////////
// LiteralExpr can be this, null, boolean, numeric, string, or regular expression literals.
type LiteralExpr struct {
TokenType
Data []byte
}
func (n LiteralExpr) String() string {
return string(n.Data)
}
// JS converts the node back to valid JavaScript
func (n LiteralExpr) JS() string {
return string(n.Data)
}
// JSON converts the node back to valid JSON
func (n LiteralExpr) JSON() (string, error) {
if n.TokenType == TrueToken || n.TokenType == FalseToken || n.TokenType == NullToken || n.TokenType == DecimalToken {
return string(n.Data), nil
} else if n.TokenType == StringToken {
if n.Data[0] == '\'' {
n.Data[0] = '"'
n.Data[len(n.Data)-1] = '"'
}
return string(n.Data), nil
}
return "", ErrInvalidJSON
}
// Element is an array literal element.
type Element struct {
Value IExpr // can be nil
Spread bool
}
func (n Element) String() string {
s := ""
if n.Value != nil {
if n.Spread {
s += "..."
}
s += n.Value.String()
}
return s
}
// JS converts the node back to valid JavaScript
func (n Element) JS() string {
s := ""
if n.Value != nil {
if n.Spread {
s += "..."
}
s += n.Value.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n Element) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ArrayExpr is an array literal.
type ArrayExpr struct {
List []Element
}
func (n ArrayExpr) String() string {
s := "["
for i, item := range n.List {
if i != 0 {
s += ", "
}
if item.Value != nil {
if item.Spread {
s += "..."
}
s += item.Value.String()
}
}
if 0 < len(n.List) && n.List[len(n.List)-1].Value == nil {
s += ","
}
return s + "]"
}
// JS converts the node back to valid JavaScript
func (n ArrayExpr) JS() string {
s := "["
for i, item := range n.List {
if i != 0 {
s += ", "
}
if item.Value != nil {
if item.Spread {
s += "..."
}
s += item.Value.JS()
}
}
if 0 < len(n.List) && n.List[len(n.List)-1].Value == nil {
s += ","
}
return s + "]"
}
// JSON converts the node back to valid JSON
func (n ArrayExpr) JSON() (string, error) {
s := "["
for i, item := range n.List {
if i != 0 {
s += ", "
}
if item.Value != nil {
if item.Spread {
return "", ErrInvalidJSON
}
ss, err := item.Value.JSON()
if err != nil {
return "", err
}
s += ss
}
}
if 0 < len(n.List) && n.List[len(n.List)-1].Value == nil {
return "", ErrInvalidJSON
}
return s + "]", nil
}
// Property is a property definition in an object literal.
type Property struct {
// either Name or Spread are set. When Spread is set then Value is AssignmentExpression
// if Init is set then Value is IdentifierReference, otherwise it can also be MethodDefinition
Name *PropertyName // can be nil
Spread bool
Value IExpr
Init IExpr // can be nil
}
func (n Property) String() string {
s := ""
if n.Name != nil {
if v, ok := n.Value.(*Var); !ok || !n.Name.IsIdent(v.Data) {
s += n.Name.String() + ": "
}
} else if n.Spread {
s += "..."
}
s += n.Value.String()
if n.Init != nil {
s += " = " + n.Init.String()
}
return s
}
// JS converts the node back to valid JavaScript
func (n Property) JS() string {
s := ""
if n.Name != nil {
if v, ok := n.Value.(*Var); !ok || !n.Name.IsIdent(v.Data) {
s += n.Name.JS() + ": "
}
} else if n.Spread {
s += "..."
}
s += n.Value.JS()
if n.Init != nil {
s += " = " + n.Init.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n Property) JSON() (string, error) {
s := ""
if n.Name == nil || n.Name.Literal.TokenType != StringToken && n.Name.Literal.TokenType != IdentifierToken || n.Spread || n.Init != nil {
return "", ErrInvalidJSON
} else if n.Name.Literal.TokenType == IdentifierToken {
n.Name.Literal.TokenType = StringToken
n.Name.Literal.Data = append(append([]byte{'"'}, n.Name.Literal.Data...), '"')
}
ss, _ := n.Name.Literal.JSON()
s += ss + ": "
var err error
ss, err = n.Value.JSON()
if err != nil {
fmt.Println("b")
return "", err
}
s += ss
return s, nil
}
// ObjectExpr is an object literal.
type ObjectExpr struct {
List []Property
}
func (n ObjectExpr) String() string {
s := "{"
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.String()
}
return s + "}"
}
// JS converts the node back to valid JavaScript
func (n ObjectExpr) JS() string {
s := "{"
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.JS()
}
return s + "}"
}
// JSON converts the node back to valid JSON
func (n ObjectExpr) JSON() (string, error) {
s := "{"
for i, item := range n.List {
if i != 0 {
s += ", "
}
ss, err := item.JSON()
if err != nil {
return "", err
}
s += ss
}
return s + "}", nil
}
// TemplatePart is a template head or middle.
type TemplatePart struct {
Value []byte
Expr IExpr
}
func (n TemplatePart) String() string {
return string(n.Value) + n.Expr.String()
}
// JS converts the node back to valid JavaScript
func (n TemplatePart) JS() string {
return string(n.Value) + n.Expr.JS()
}
// JSON converts the node back to valid JSON
func (n TemplatePart) JSON() (string, error) {
return "", ErrInvalidJSON
}
// TemplateExpr is a template literal or member/call expression, super property, or optional chain with template literal.
type TemplateExpr struct {
Tag IExpr // can be nil
List []TemplatePart
Tail []byte
Prec OpPrec
}
func (n TemplateExpr) String() string {
s := ""
if n.Tag != nil {
s += n.Tag.String()
}
for _, item := range n.List {
s += item.String()
}
return s + string(n.Tail)
}
// JS converts the node back to valid JavaScript
func (n TemplateExpr) JS() string {
s := ""
if n.Tag != nil {
s += n.Tag.JS()
}
for _, item := range n.List {
s += item.JS()
}
return s + string(n.Tail)
}
// JSON converts the node back to valid JSON
func (n TemplateExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// GroupExpr is a parenthesized expression.
type GroupExpr struct {
X IExpr
}
func (n GroupExpr) String() string {
return "(" + n.X.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n GroupExpr) JS() string {
return "(" + n.X.JS() + ")"
}
// JSON converts the node back to valid JSON
func (n GroupExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// IndexExpr is a member/call expression, super property, or optional chain with an index expression.
type IndexExpr struct {
X IExpr
Y IExpr
Prec OpPrec
}
func (n IndexExpr) String() string {
return "(" + n.X.String() + "[" + n.Y.String() + "])"
}
// JS converts the node back to valid JavaScript
func (n IndexExpr) JS() string {
return n.X.JS() + "[" + n.Y.JS() + "]"
}
// JSON converts the node back to valid JSON
func (n IndexExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// DotExpr is a member/call expression, super property, or optional chain with a dot expression.
type DotExpr struct {
X IExpr
Y LiteralExpr
Prec OpPrec
}
func (n DotExpr) String() string {
return "(" + n.X.String() + "." + n.Y.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n DotExpr) JS() string {
return n.X.JS() + "." + n.Y.JS()
}
// JSON converts the node back to valid JSON
func (n DotExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// NewTargetExpr is a new target meta property.
type NewTargetExpr struct {
}
func (n NewTargetExpr) String() string {
return "(new.target)"
}
// JS converts the node back to valid JavaScript
func (n NewTargetExpr) JS() string {
return "new.target"
}
// JSON converts the node back to valid JSON
func (n NewTargetExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ImportMetaExpr is a import meta meta property.
type ImportMetaExpr struct {
}
func (n ImportMetaExpr) String() string {
return "(import.meta)"
}
// JS converts the node back to valid JavaScript
func (n ImportMetaExpr) JS() string {
return "import.meta"
}
// JSON converts the node back to valid JSON
func (n ImportMetaExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
type Arg struct {
Value IExpr
Rest bool
}
func (n Arg) String() string {
s := ""
if n.Rest {
s += "..."
}
return s + n.Value.String()
}
// JS converts the node back to valid JavaScript
func (n Arg) JS() string {
s := ""
if n.Rest {
s += "..."
}
return s + n.Value.JS()
}
// JSON converts the node back to valid JSON
func (n Arg) JSON() (string, error) {
return "", ErrInvalidJSON
}
// Args is a list of arguments as used by new and call expressions.
type Args struct {
List []Arg
}
func (n Args) String() string {
s := "("
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.String()
}
return s + ")"
}
// JS converts the node back to valid JavaScript
func (n Args) JS() string {
s := ""
for i, item := range n.List {
if i != 0 {
s += ", "
}
s += item.JS()
}
return s
}
// JSON converts the node back to valid JSON
func (n Args) JSON() (string, error) {
return "", ErrInvalidJSON
}
// NewExpr is a new expression or new member expression.
type NewExpr struct {
X IExpr
Args *Args // can be nil
}
func (n NewExpr) String() string {
if n.Args != nil {
return "(new " + n.X.String() + n.Args.String() + ")"
}
return "(new " + n.X.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n NewExpr) JS() string {
if n.Args != nil {
return "new " + n.X.JS() + "(" + n.Args.JS() + ")"
}
// always use parentheses to prevent errors when chaining e.g. new Date().getTime()
return "new " + n.X.JS() + "()"
}
// JSON converts the node back to valid JSON
func (n NewExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// CallExpr is a call expression.
type CallExpr struct {
X IExpr
Args Args
}
func (n CallExpr) String() string {
return "(" + n.X.String() + n.Args.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n CallExpr) JS() string {
return n.X.JS() + "(" + n.Args.JS() + ")"
}
// JSON converts the node back to valid JSON
func (n CallExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// OptChainExpr is an optional chain.
type OptChainExpr struct {
X IExpr
Y IExpr // can be CallExpr, IndexExpr, LiteralExpr, or TemplateExpr
}
func (n OptChainExpr) String() string {
s := "(" + n.X.String() + "?."
switch y := n.Y.(type) {
case *CallExpr:
return s + y.Args.String() + ")"
case *IndexExpr:
return s + "[" + y.Y.String() + "])"
default:
return s + y.String() + ")"
}
}
// JS converts the node back to valid JavaScript
func (n OptChainExpr) JS() string {
s := n.X.String() + "?."
switch y := n.Y.(type) {
case *CallExpr:
return s + y.Args.JS() + ")"
case *IndexExpr:
return s + "[" + y.Y.JS() + "])"
default:
return s + y.JS()
}
}
// JSON converts the node back to valid JSON
func (n OptChainExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// UnaryExpr is an update or unary expression.
type UnaryExpr struct {
Op TokenType
X IExpr
}
func (n UnaryExpr) String() string {
if n.Op == PostIncrToken || n.Op == PostDecrToken {
return "(" + n.X.String() + n.Op.String() + ")"
} else if IsIdentifierName(n.Op) {
return "(" + n.Op.String() + " " + n.X.String() + ")"
}
return "(" + n.Op.String() + n.X.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n UnaryExpr) JS() string {
if n.Op == PostIncrToken || n.Op == PostDecrToken {
return n.X.JS() + n.Op.String()
} else if IsIdentifierName(n.Op) {
return n.Op.String() + " " + n.X.JS()
}
return n.Op.String() + n.X.JS()
}
// JSON converts the node back to valid JSON
func (n UnaryExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// BinaryExpr is a binary expression.
type BinaryExpr struct {
Op TokenType
X, Y IExpr
}
func (n BinaryExpr) String() string {
if IsIdentifierName(n.Op) {
return "(" + n.X.String() + " " + n.Op.String() + " " + n.Y.String() + ")"
}
return "(" + n.X.String() + n.Op.String() + n.Y.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n BinaryExpr) JS() string {
return n.X.JS() + " " + n.Op.String() + " " + n.Y.JS()
}
// JSON converts the node back to valid JSON
func (n BinaryExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// CondExpr is a conditional expression.
type CondExpr struct {
Cond, X, Y IExpr
}
func (n CondExpr) String() string {
return "(" + n.Cond.String() + " ? " + n.X.String() + " : " + n.Y.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n CondExpr) JS() string {
return n.Cond.JS() + " ? " + n.X.JS() + " : " + n.Y.JS()
}
// JSON converts the node back to valid JSON
func (n CondExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// YieldExpr is a yield expression.
type YieldExpr struct {
Generator bool
X IExpr // can be nil
}
func (n YieldExpr) String() string {
if n.X == nil {
return "(yield)"
}
s := "(yield"
if n.Generator {
s += "*"
}
return s + " " + n.X.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n YieldExpr) JS() string {
if n.X == nil {
return "yield"
}
s := "yield"
if n.Generator {
s += "*"
}
return s + " " + n.X.JS()
}
// JSON converts the node back to valid JSON
func (n YieldExpr) JSON() (string, error) {
return "", ErrInvalidJSON
}
// ArrowFunc is an (async) arrow function.
type ArrowFunc struct {
Async bool
Params Params
Body BlockStmt
}
func (n ArrowFunc) String() string {
s := "("
if n.Async {
s += "async "
}
return s + n.Params.String() + " => " + n.Body.String() + ")"
}
// JS converts the node back to valid JavaScript
func (n ArrowFunc) JS() string {
s := ""
if n.Async {
s += "async "
}
return s + n.Params.JS() + " => " + n.Body.JS()
}
// JSON converts the node back to valid JSON
func (n ArrowFunc) JSON() (string, error) {
return "", ErrInvalidJSON
}
func (v *Var) exprNode() {}
func (n LiteralExpr) exprNode() {}
func (n ArrayExpr) exprNode() {}
func (n ObjectExpr) exprNode() {}
func (n TemplateExpr) exprNode() {}
func (n GroupExpr) exprNode() {}
func (n DotExpr) exprNode() {}
func (n IndexExpr) exprNode() {}
func (n NewTargetExpr) exprNode() {}
func (n ImportMetaExpr) exprNode() {}
func (n NewExpr) exprNode() {}
func (n CallExpr) exprNode() {}
func (n OptChainExpr) exprNode() {}
func (n UnaryExpr) exprNode() {}
func (n BinaryExpr) exprNode() {}
func (n CondExpr) exprNode() {}
func (n YieldExpr) exprNode() {}
func (n ArrowFunc) exprNode() {}
parse-2.5.21/js/benchmark_test.go 0000664 0000000 0000000 00000016167 14117006534 0016663 0 ustar 00root root 0000000 0000000 package js
import (
"bytes"
"fmt"
"math/rand"
"testing"
)
var z = 0
var n = []int{4, 12, 20, 30, 40, 50, 60, 150}
var randStrings [][]byte
var mapStrings []map[string]bool
var mapStructStrings []map[string]struct{}
var mapInts []map[int]bool
var arrayStrings [][]string
var arrayBytes [][][]byte
var arrayInts [][]int
var identifiers [][]byte
func helperRandString() string {
cs := []byte("abcdefghijklmnopqrstuvwxyz")
b := make([]byte, rand.Intn(10))
for i := range b {
b[i] = cs[rand.Intn(len(cs))]
}
return string(b)
}
func init() {
for j := 0; j < len(n); j++ {
ms := map[string]bool{}
mss := map[string]struct{}{}
mi := map[int]bool{}
as := []string{}
ab := [][]byte{}
ai := []int{}
for i := 0; i < n[j]; i++ {
s := helperRandString()
ms[s] = true
mss[s] = struct{}{}
mi[i] = true
as = append(as, s)
ab = append(ab, []byte(s))
ai = append(ai, i)
}
mapStrings = append(mapStrings, ms)
mapStructStrings = append(mapStructStrings, mss)
mapInts = append(mapInts, mi)
arrayStrings = append(arrayStrings, as)
arrayBytes = append(arrayBytes, ab)
arrayInts = append(arrayInts, ai)
}
keywords := []string{"as", "for", "while", "extends", "do", "else", "static"}
for j := 0; j < 1000; j++ {
randStrings = append(randStrings, []byte(helperRandString()))
if rand.Float64() < 0.5 {
identifiers = append(identifiers, []byte(keywords[rand.Intn(len(keywords))]))
} else {
identifiers = append(identifiers, []byte(helperRandString()))
}
}
}
func BenchmarkAddMapStrings(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
m := map[string]bool{}
for i := 0; i < n[j]; i++ {
m[arrayStrings[j][i]] = true
}
}
})
}
}
func BenchmarkAddMapInts(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
m := map[int]bool{}
for i := 0; i < n[j]; i++ {
m[arrayInts[j][i]] = true
}
}
})
}
}
func BenchmarkAddArrayStrings(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
a := []string{}
for i := 0; i < n[j]; i++ {
a = append(a, arrayStrings[j][i])
}
}
})
}
}
func BenchmarkAddArrayBytes(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
a := [][]byte{}
for i := 0; i < n[j]; i++ {
a = append(a, arrayBytes[j][i])
}
}
})
}
}
func BenchmarkAddArrayInts(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
a := []int{}
for i := 0; i < n[j]; i++ {
a = append(a, arrayInts[j][i])
}
}
})
}
}
func BenchmarkLookupMapStrings(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
if mapStrings[j][arrayStrings[j][i]] == true {
z++
}
}
}
})
}
}
func BenchmarkLookupMapStructStrings(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
if _, ok := mapStructStrings[j][string(arrayBytes[j][i])]; ok {
z++
}
}
}
})
}
}
func BenchmarkLookupMapBytes(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
if mapStrings[j][string(arrayBytes[j][i])] == true {
z++
}
}
}
})
}
}
func BenchmarkLookupMapInts(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
if mapInts[j][arrayInts[j][i]] == true {
z++
}
}
}
})
}
}
func BenchmarkLookupArrayStrings(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
s := arrayStrings[j][i]
for _, ss := range arrayStrings[j] {
if s == ss {
z++
break
}
}
}
}
})
}
}
func BenchmarkLookupArrayBytes(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
s := arrayBytes[j][i]
for _, ss := range arrayBytes[j] {
if bytes.Equal(s, ss) {
z++
break
}
}
}
}
})
}
}
func BenchmarkLookupArrayInts(b *testing.B) {
for j := 0; j < len(n); j++ {
b.Run(fmt.Sprintf("%v", n[j]), func(b *testing.B) {
for k := 0; k < b.N; k++ {
for i := 0; i < n[j]; i++ {
q := arrayInts[j][i]
for _, qq := range arrayInts[j] {
if q == qq {
z++
break
}
}
}
}
})
}
}
type benchRef uint
type benchVar struct {
ptr *benchVar
data []byte
}
var listAST []interface{}
var listPtr []*benchVar
var listVar []benchVar
func BenchmarkASTPtr(b *testing.B) {
for j := 0; j < 3; j++ {
b.Run(fmt.Sprintf("%v", j), func(b *testing.B) {
for k := 0; k < b.N; k++ {
listAST = listAST[:0:0]
listPtr = listPtr[:0:0]
for _, b := range randStrings {
v := &benchVar{nil, b}
listAST = append(listAST, v)
listPtr = append(listPtr, v)
}
}
})
}
}
func BenchmarkASTIdx(b *testing.B) {
for j := 0; j < 3; j++ {
b.Run(fmt.Sprintf("%v", j), func(b *testing.B) {
for k := 0; k < b.N; k++ {
listAST = listAST[:0:0]
listPtr = listPtr[:0:0]
listVar = listVar[:0:0]
for _, b := range randStrings {
listVar = append(listVar, benchVar{nil, b})
v := &listVar[len(listVar)-1]
ref := benchRef(len(listVar) - 1)
listAST = append(listAST, ref)
listPtr = append(listPtr, v)
}
}
})
}
}
var listInterface []interface{}
var listVars []*Var
func BenchmarkInterfaceAddPtr(b *testing.B) {
listInterface = listInterface[:0:0]
for k := 0; k < b.N; k++ {
v := &Var{nil, nil, 0, 0}
listInterface = append(listInterface, v)
}
}
//func BenchmarkInterfaceAddVal32(b *testing.B) {
// listInterface = listInterface[:0:0]
// for k := 0; k < b.N; k++ {
// v := &Var{0, nil, nil, 0}
// listInterface = append(listInterface, v.Ref)
// }
//}
//
//func BenchmarkInterfaceAddVal64(b *testing.B) {
// listInterface = listInterface[:0:0]
// for k := 0; k < b.N; k++ {
// v := &Var{0, nil, nil, 0}
// listInterface = append(listInterface, uint64(v.Ref))
// }
//}
func BenchmarkInterfaceCheckPtr(b *testing.B) {
v := &Var{nil, nil, 0, 0}
i := interface{}(v)
for k := 0; k < b.N; k++ {
if r, ok := i.(*Var); ok {
_ = r
z++
}
}
}
//func BenchmarkInterfaceCheckVal(b *testing.B) {
// ref := VarRef(300)
// i := interface{}(ref)
// for k := 0; k < b.N; k++ {
// if r, ok := i.(VarRef); ok {
// _ = r
// z++
// }
// }
//}
////////////////////////////////////////////////////////////////
func BenchmarkMap(b *testing.B) {
for k := 0; k < b.N; k++ {
for _, ident := range identifiers {
_ = Keywords[string(ident)]
}
}
}
parse-2.5.21/js/js_test.go 0000664 0000000 0000000 00000017575 14117006534 0015351 0 ustar 00root root 0000000 0000000 package js
import (
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
func TestJS(t *testing.T) {
var tests = []struct {
js string
expected string
}{
// BlockStmt
{"if (true) { x(1, 2, 3); };", "if (true) { x(1, 2, 3); }; "},
// IfStmt
{"if (true) { true; };", "if (true) { true; }; "},
{"if (true) { true; } else { false; };", "if (true) { true; } else { false; }; "},
{"if (true) { true; } else { if(true) { true; } else { false; } };", "if (true) { true; } else { if (true) { true; } else { false; }; }; "},
// DoWhileStmt
{"do { continue; } while (true);", "do { continue; } while (true); "},
{"do { x = 1; } while (true);", "do { x = 1; } while (true); "},
// WhileStmt
{"while (true) { true; };", "while (true) { true; }; "},
{"while (true) { x = 1; };", "while (true) { x = 1; }; "},
// ForStmt
{"for ( ; ; ) { true; };", "for ( ; ; ) { true; }; "},
{"for (x = 1; ; ) { true; };", "for (x = 1; ; ) { true; }; "},
{"for (x = 1; x < 2; ) { true; };", "for (x = 1; x < 2; ) { true; }; "},
{"for (x = 1; x < 2; x++) { true; };", "for (x = 1; x < 2; x++) { true; }; "},
{"for (x = 1; x < 2; x++) { x = 1; };", "for (x = 1; x < 2; x++) { x = 1; }; "},
// ForInStmt
{"for (var x in [1, 2]) { true; };", "for (var x in [1, 2]) { true; }; "},
{"for (var x in [1, 2]) { x = 1; };", "for (var x in [1, 2]) { x = 1; }; "},
// ForOfStmt
{"for (const element of [1, 2]) { true; };", "for (const element of [1, 2]) { true; }; "},
{"for (const element of [1, 2]) { x = 1; };", "for (const element of [1, 2]) { x = 1; }; "},
// SwitchStmt
{"switch (true) { case true: break; case false: false; };", "switch (true) { case true: break; case false: false; }; "},
{"switch (true) { case true: x(); break; case false: x(); false; };", "switch (true) { case true: x(); break; case false: x(); false; }; "},
{"switch (true) { default: false; };", "switch (true) { default: false; }; "},
// BranchStmt
{"for (i = 0; i < 3; i++) { continue; }; ", "for (i = 0; i < 3; i++) { continue; }; "},
{"for (i = 0; i < 3; i++) { x = 1; }; ", "for (i = 0; i < 3; i++) { x = 1; }; "},
// ReturnStmt
{"return;", "return; "},
{"return 1;", "return 1; "},
// WithStmt
{"with (true) { true; };", "with (true) { true; }; "},
{"with (true) { x = 1; };", "with (true) { x = 1; }; "},
// LabelledStmt
{"loop: for (x = 0; x < 1; x++) { true; };", "loop: for (x = 0; x < 1; x++) { true; }; "},
// ThrowStmt
{"throw x;", "throw x; "},
// TryStmt
{"try { true; } catch(e) { };", "try { true; } catch(e) { }; "},
{"try { true; } catch(e) { true; };", "try { true; } catch(e) { true; }; "},
{"try { true; } catch(e) { x = 1; };", "try { true; } catch(e) { x = 1; }; "},
// DebuggerStmt
{"debugger;", "debugger; "},
// Alias
{"import * as name from 'module-name';", "import * as name from 'module-name'; "},
// ImportStmt
{"import defaultExport from 'module-name';", "import defaultExport from 'module-name'; "},
{"import * as name from 'module-name';", "import * as name from 'module-name'; "},
{"import { export1 } from 'module-name';", "import { export1 } from 'module-name'; "},
{"import { export1 as alias1 } from 'module-name';", "import { export1 as alias1 } from 'module-name'; "},
{"import { export1 , export2 } from 'module-name';", "import { export1 , export2 } from 'module-name'; "},
{"import { foo , bar } from 'module-name/path/to/specific/un-exported/file';", "import { foo , bar } from 'module-name/path/to/specific/un-exported/file'; "},
{"import defaultExport, * as name from 'module-name';", "import defaultExport , * as name from 'module-name'; "},
{"import 'module-name';", "import 'module-name'; "},
{"var promise = import('module-name');", "var promise = import('module-name'); "},
// ExportStmt
{"export { myFunction as default };", "export { myFunction as default }; "},
{"export default k = 12;", "export default k = 12; "},
// DirectivePrologueStmt
{"'use strict';", "'use strict'; "},
// BindingArray
{"let [name = 5] = z;", "let [name = 5] = z; "},
// BindingObject
{"let {} = z;", "let { } = z; "},
// BindingElement
{"let [name = 5] = z;", "let [name = 5] = z; "},
// VarDecl
{"x = 1;", "x = 1; "},
{"var x;", "var x; "},
{"var x = 1;", "var x = 1; "},
{"var x, y = [];", "var x, y = []; "},
{"let x;", "let x; "},
{"let x = 1;", "let x = 1; "},
{"const x = 1;", "const x = 1; "},
// Params
{"function xyz (a, b) { };", "function xyz (a, b) { }; "},
{"function xyz (a, b, ...c) { };", "function xyz (a, b, ...c) { }; "},
// FuncDecl
{"function xyz (a, b) { };", "function xyz (a, b) { }; "},
// MethodDecl
{"class A { field; static get method () { }; };", "class A { field; static get method () { }; }; "},
// FieldDefinition
{"class A { field; };", "class A { field; }; "},
{"class A { field = 5; };", "class A { field = 5; }; "},
// ClassDecl
{"class A { field; static get method () { }; };", "class A { field; static get method () { }; }; "},
{"class B extends A { field; static get method () { }; };", "class B extends A { field; static get method () { }; }; "},
// LiteralExpr
{"'test';", "'test'; "},
// ArrayExpr
{"[1, 2, 3];", "[1, 2, 3]; "},
// Property
{`x = {x: "value"};`, `x = {x: "value"}; `},
{`x = {"x": "value"};`, `x = {x: "value"}; `},
{`x = {"1a": 2};`, `x = {"1a": 2}; `},
// ObjectExpr
{`x = {x: "value", y: "value"};`, `x = {x: "value", y: "value"}; `},
// TemplateExpr
{"x = `value`;", "x = `value`; "},
{"x = `value${'hi'}`;", "x = `value${'hi'}`; "},
// GroupExpr
{"x = (1 + 1) / 1;", "x = (1 + 1) / 1; "},
// IndexExpr
{"x = y[1];", "x = y[1]; "},
// DotExpr
{"x = y.z;", "x = y.z; "},
// NewTargetExpr
{"x = new.target;", "x = new.target; "},
// ImportMetaExpr
{"x = import.meta;", "x = import.meta; "},
// Args
{"x(1, 2);", "x(1, 2); "},
// NewExpr
{"new x;", "new x(); "},
{"new x(1);", "new x(1); "},
{"new Date().getTime();", "new Date().getTime(); "},
// CallExpr
{"x();", "x(); "},
// OptChainExpr
{"x = y?.z;", "x = y?.z; "},
// UnaryExpr
{"x = 1 + 1;", "x = 1 + 1; "},
// BinaryExpr
{"a << b;", "a << b; "},
// CondExpr
{"a && b;", "a && b; "},
{"a || b;", "a || b; "},
// YieldExpr
{"x = function* foo (x) { while (x < 2) { yield x; x++; }; };", "x = function* foo (x) { while (x < 2) { yield x; x++; }; }; "},
// ArrowFunc
{"(x) => { y(); };", "(x) => { y(); }; "},
{"(x, y) => { z(); };", "(x, y) => { z(); }; "},
{"async (x, y) => { z(); };", "async (x, y) => { z(); }; "},
}
for _, tt := range tests {
t.Run(tt.js, func(t *testing.T) {
ast, err := Parse(parse.NewInputString(tt.js))
if err != io.EOF {
test.Error(t, err)
}
test.String(t, ast.JS(), tt.expected)
})
}
}
func TestJSRealWorldJS(t *testing.T) {
js := `
var _0x34d2=['log','415343ArZKCi','11ItVJMI','98599KfnlVw','139pQCPDx','526583DuLSJk','Hello\x20World!','5823JSTLxZ','543807ONUblA','2uewDkG','146389ygBdVV','2273BZpJsB'];(function(_0x6b2cbe,_0x3fd0f4){var _0xffa95b=_0x3d17;while(!![]){try{var _0x5239dc=-parseInt(_0xffa95b(0x132))+parseInt(_0xffa95b(0x12f))+-parseInt(_0xffa95b(0x131))*-parseInt(_0xffa95b(0x12c))+parseInt(_0xffa95b(0x135))*-parseInt(_0xffa95b(0x12e))+-parseInt(_0xffa95b(0x12d))+-parseInt(_0xffa95b(0x134))+-parseInt(_0xffa95b(0x133))*-parseInt(_0xffa95b(0x12b));if(_0x5239dc===_0x3fd0f4)break;else _0x6b2cbe['push'](_0x6b2cbe['shift']());}catch(_0x12a1ea){_0x6b2cbe['push'](_0x6b2cbe['shift']());}}}(_0x34d2,0x4d4a4));function hi(){var _0x2d5dfd=_0x3d17;console[_0x2d5dfd(0x12a)](_0x2d5dfd(0x130));}function _0x3d17(_0x59c992,_0x5be83e){_0x59c992=_0x59c992-0x12a;var _0x34d208=_0x34d2[_0x59c992];return _0x34d208;}hi();
`
ast, err := Parse(parse.NewInputString(js))
if err != nil && err != io.EOF {
t.Fatal(err)
}
// reparse to make sure JS is still valid
_, err = Parse(parse.NewInputString(ast.JS()))
if err != nil {
t.Error("Err: ", err)
}
}
parse-2.5.21/js/lex.go 0000664 0000000 0000000 00000050117 14117006534 0014453 0 ustar 00root root 0000000 0000000 // Package js is an ECMAScript5.1 lexer following the specifications at http://www.ecma-international.org/ecma-262/5.1/.
package js
import (
"unicode"
"unicode/utf8"
"github.com/tdewolff/parse/v2"
)
var identifierStart = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Other_ID_Start}
var identifierContinue = []*unicode.RangeTable{unicode.Lu, unicode.Ll, unicode.Lt, unicode.Lm, unicode.Lo, unicode.Nl, unicode.Mn, unicode.Mc, unicode.Nd, unicode.Pc, unicode.Other_ID_Continue}
// IsIdentifierStart returns true if the byte-slice start is the start of an identifier
func IsIdentifierStart(b []byte) bool {
r, _ := utf8.DecodeRune(b)
return r == '$' || r == '\\' || r == '_' || unicode.IsOneOf(identifierStart, r)
}
// IsIdentifierContinue returns true if the byte-slice start is a continuation of an identifier
func IsIdentifierContinue(b []byte) bool {
r, _ := utf8.DecodeRune(b)
return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
}
// IsIdentifierEnd returns true if the byte-slice end is a start or continuation of an identifier
func IsIdentifierEnd(b []byte) bool {
r, _ := utf8.DecodeLastRune(b)
return r == '$' || r == '\\' || r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r)
}
////////////////////////////////////////////////////////////////
// Lexer is the state for the lexer.
type Lexer struct {
r *parse.Input
err error
prevLineTerminator bool
prevNumericLiteral bool
level int
templateLevels []int
}
// NewLexer returns a new Lexer for a given io.Reader.
func NewLexer(r *parse.Input) *Lexer {
return &Lexer{
r: r,
prevLineTerminator: true,
level: 0,
templateLevels: []int{},
}
}
// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
func (l *Lexer) Err() error {
if l.err != nil {
return l.err
}
return l.r.Err()
}
// RegExp reparses the input stream for a regular expression. It is assumed that we just received DivToken or DivEqToken with Next(). This function will go back and read that as a regular expression.
func (l *Lexer) RegExp() (TokenType, []byte) {
if 0 < l.r.Offset() && l.r.Peek(-1) == '/' {
l.r.Move(-1)
} else if 1 < l.r.Offset() && l.r.Peek(-1) == '=' && l.r.Peek(-2) == '/' {
l.r.Move(-2)
} else {
l.err = parse.NewErrorLexer(l.r, "expected / or /=")
return ErrorToken, nil
}
l.r.Skip() // trick to set start = pos
if l.consumeRegExpToken() {
return RegExpToken, l.r.Shift()
}
l.err = parse.NewErrorLexer(l.r, "unexpected EOF or newline")
return ErrorToken, nil
}
// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (l *Lexer) Next() (TokenType, []byte) {
prevLineTerminator := l.prevLineTerminator
l.prevLineTerminator = false
prevNumericLiteral := l.prevNumericLiteral
l.prevNumericLiteral = false
// study on 50x jQuery shows:
// spaces: 20k
// alpha: 16k
// newlines: 14.4k
// operators: 4k
// numbers and dot: 3.6k
// (): 3.4k
// {}: 1.8k
// []: 0.9k
// "': 1k
// semicolon: 2.4k
// colon: 0.8k
// comma: 2.4k
// slash: 1.4k
// `~: almost 0
c := l.r.Peek(0)
switch c {
case ' ', '\t', '\v', '\f':
l.r.Move(1)
for l.consumeWhitespace() {
}
l.prevLineTerminator = prevLineTerminator
return WhitespaceToken, l.r.Shift()
case '\n', '\r':
l.r.Move(1)
for l.consumeLineTerminator() {
}
l.prevLineTerminator = true
return LineTerminatorToken, l.r.Shift()
case '>', '=', '!', '+', '*', '%', '&', '|', '^', '~', '?':
if tt := l.consumeOperatorToken(); tt != ErrorToken {
return tt, l.r.Shift()
}
case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9', '.':
if tt := l.consumeNumericToken(); tt != ErrorToken || l.r.Pos() != 0 {
l.prevNumericLiteral = true
return tt, l.r.Shift()
} else if c == '.' {
l.r.Move(1)
if l.r.Peek(0) == '.' && l.r.Peek(1) == '.' {
l.r.Move(2)
return EllipsisToken, l.r.Shift()
}
return DotToken, l.r.Shift()
}
case ',':
l.r.Move(1)
return CommaToken, l.r.Shift()
case ';':
l.r.Move(1)
return SemicolonToken, l.r.Shift()
case '(':
l.level++
l.r.Move(1)
return OpenParenToken, l.r.Shift()
case ')':
l.level--
l.r.Move(1)
return CloseParenToken, l.r.Shift()
case '/':
if tt := l.consumeCommentToken(); tt != ErrorToken {
return tt, l.r.Shift()
} else if tt := l.consumeOperatorToken(); tt != ErrorToken {
return tt, l.r.Shift()
}
case '{':
l.level++
l.r.Move(1)
return OpenBraceToken, l.r.Shift()
case '}':
l.level--
if len(l.templateLevels) != 0 && l.level == l.templateLevels[len(l.templateLevels)-1] {
return l.consumeTemplateToken(), l.r.Shift()
}
l.r.Move(1)
return CloseBraceToken, l.r.Shift()
case ':':
l.r.Move(1)
return ColonToken, l.r.Shift()
case '\'', '"':
if l.consumeStringToken() {
return StringToken, l.r.Shift()
}
case ']':
l.r.Move(1)
return CloseBracketToken, l.r.Shift()
case '[':
l.r.Move(1)
return OpenBracketToken, l.r.Shift()
case '<', '-':
if l.consumeHTMLLikeCommentToken(prevLineTerminator) {
return CommentToken, l.r.Shift()
} else if tt := l.consumeOperatorToken(); tt != ErrorToken {
return tt, l.r.Shift()
}
case '`':
l.templateLevels = append(l.templateLevels, l.level)
return l.consumeTemplateToken(), l.r.Shift()
case '#':
l.r.Move(1)
if l.consumeIdentifierToken() {
return PrivateIdentifierToken, l.r.Shift()
}
return ErrorToken, nil
default:
if l.consumeIdentifierToken() {
if prevNumericLiteral {
l.err = parse.NewErrorLexer(l.r, "unexpected identifier after number")
return ErrorToken, nil
} else if keyword, ok := Keywords[string(l.r.Lexeme())]; ok {
return keyword, l.r.Shift()
}
return IdentifierToken, l.r.Shift()
}
if 0xC0 <= c {
if l.consumeWhitespace() {
for l.consumeWhitespace() {
}
l.prevLineTerminator = prevLineTerminator
return WhitespaceToken, l.r.Shift()
} else if l.consumeLineTerminator() {
for l.consumeLineTerminator() {
}
l.prevLineTerminator = true
return LineTerminatorToken, l.r.Shift()
}
} else if c == 0 && l.r.Err() != nil {
return ErrorToken, nil
}
}
r, _ := l.r.PeekRune(0)
l.err = parse.NewErrorLexer(l.r, "unexpected %s", parse.Printable(r))
return ErrorToken, l.r.Shift()
}
////////////////////////////////////////////////////////////////
/*
The following functions follow the specifications at http://www.ecma-international.org/ecma-262/5.1/
*/
func (l *Lexer) consumeWhitespace() bool {
c := l.r.Peek(0)
if c == ' ' || c == '\t' || c == '\v' || c == '\f' {
l.r.Move(1)
return true
} else if 0xC0 <= c {
if r, n := l.r.PeekRune(0); r == '\u00A0' || r == '\uFEFF' || unicode.Is(unicode.Zs, r) {
l.r.Move(n)
return true
}
}
return false
}
func (l *Lexer) isLineTerminator() bool {
c := l.r.Peek(0)
if c == '\n' || c == '\r' {
return true
} else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
return true
}
return false
}
func (l *Lexer) consumeLineTerminator() bool {
c := l.r.Peek(0)
if c == '\n' {
l.r.Move(1)
return true
} else if c == '\r' {
if l.r.Peek(1) == '\n' {
l.r.Move(2)
} else {
l.r.Move(1)
}
return true
} else if c == 0xE2 && l.r.Peek(1) == 0x80 && (l.r.Peek(2) == 0xA8 || l.r.Peek(2) == 0xA9) {
l.r.Move(3)
return true
}
return false
}
func (l *Lexer) consumeDigit() bool {
if c := l.r.Peek(0); c >= '0' && c <= '9' {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeHexDigit() bool {
if c := l.r.Peek(0); (c >= '0' && c <= '9') || (c >= 'a' && c <= 'f') || (c >= 'A' && c <= 'F') {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeBinaryDigit() bool {
if c := l.r.Peek(0); c == '0' || c == '1' {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeOctalDigit() bool {
if c := l.r.Peek(0); c >= '0' && c <= '7' {
l.r.Move(1)
return true
}
return false
}
func (l *Lexer) consumeUnicodeEscape() bool {
if l.r.Peek(0) != '\\' || l.r.Peek(1) != 'u' {
return false
}
mark := l.r.Pos()
l.r.Move(2)
if c := l.r.Peek(0); c == '{' {
l.r.Move(1)
if l.consumeHexDigit() {
for l.consumeHexDigit() {
}
if c := l.r.Peek(0); c == '}' {
l.r.Move(1)
return true
}
}
l.r.Rewind(mark)
return false
} else if !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() || !l.consumeHexDigit() {
l.r.Rewind(mark)
return false
}
return true
}
func (l *Lexer) consumeSingleLineComment() {
for {
c := l.r.Peek(0)
if c == '\r' || c == '\n' || c == 0 && l.r.Err() != nil {
break
} else if 0xC0 <= c {
if r, _ := l.r.PeekRune(0); r == '\u2028' || r == '\u2029' {
break
}
}
l.r.Move(1)
}
}
////////////////////////////////////////////////////////////////
func (l *Lexer) consumeHTMLLikeCommentToken(prevLineTerminator bool) bool {
c := l.r.Peek(0)
if c == '<' && l.r.Peek(1) == '!' && l.r.Peek(2) == '-' && l.r.Peek(3) == '-' {
// opening HTML-style single line comment
l.r.Move(4)
l.consumeSingleLineComment()
return true
} else if prevLineTerminator && c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
// closing HTML-style single line comment
// (only if current line didn't contain any meaningful tokens)
l.r.Move(3)
l.consumeSingleLineComment()
return true
}
return false
}
func (l *Lexer) consumeCommentToken() TokenType {
c := l.r.Peek(1)
if c == '/' {
// single line comment
l.r.Move(2)
l.consumeSingleLineComment()
return CommentToken
} else if c == '*' {
l.r.Move(2)
tt := CommentToken
for {
c := l.r.Peek(0)
if c == '*' && l.r.Peek(1) == '/' {
l.r.Move(2)
break
} else if c == 0 && l.r.Err() != nil {
break
} else if l.consumeLineTerminator() {
l.prevLineTerminator = true
tt = CommentLineTerminatorToken
} else {
l.r.Move(1)
}
}
return tt
}
return ErrorToken
}
var opTokens = map[byte]TokenType{
'=': EqToken,
'!': NotToken,
'<': LtToken,
'>': GtToken,
'+': AddToken,
'-': SubToken,
'*': MulToken,
'/': DivToken,
'%': ModToken,
'&': BitAndToken,
'|': BitOrToken,
'^': BitXorToken,
'~': BitNotToken,
'?': QuestionToken,
}
var opEqTokens = map[byte]TokenType{
'=': EqEqToken,
'!': NotEqToken,
'<': LtEqToken,
'>': GtEqToken,
'+': AddEqToken,
'-': SubEqToken,
'*': MulEqToken,
'/': DivEqToken,
'%': ModEqToken,
'&': BitAndEqToken,
'|': BitOrEqToken,
'^': BitXorEqToken,
}
var opOpTokens = map[byte]TokenType{
'<': LtLtToken,
'+': IncrToken,
'-': DecrToken,
'*': ExpToken,
'&': AndToken,
'|': OrToken,
'?': NullishToken,
}
var opOpEqTokens = map[byte]TokenType{
'<': LtLtEqToken,
'*': ExpEqToken,
'&': AndEqToken,
'|': OrEqToken,
'?': NullishEqToken,
}
func (l *Lexer) consumeOperatorToken() TokenType {
c := l.r.Peek(0)
l.r.Move(1)
if l.r.Peek(0) == '=' {
l.r.Move(1)
if l.r.Peek(0) == '=' && (c == '!' || c == '=') {
l.r.Move(1)
if c == '!' {
return NotEqEqToken
}
return EqEqEqToken
}
return opEqTokens[c]
} else if l.r.Peek(0) == c && (c == '+' || c == '-' || c == '*' || c == '&' || c == '|' || c == '?' || c == '<') {
l.r.Move(1)
if l.r.Peek(0) == '=' && c != '+' && c != '-' {
l.r.Move(1)
return opOpEqTokens[c]
}
return opOpTokens[c]
} else if c == '?' && l.r.Peek(0) == '.' && (l.r.Peek(1) < '0' || l.r.Peek(1) > '9') {
l.r.Move(1)
return OptChainToken
} else if c == '=' && l.r.Peek(0) == '>' {
l.r.Move(1)
return ArrowToken
} else if c == '>' && l.r.Peek(0) == '>' {
l.r.Move(1)
if l.r.Peek(0) == '>' {
l.r.Move(1)
if l.r.Peek(0) == '=' {
l.r.Move(1)
return GtGtGtEqToken
}
return GtGtGtToken
} else if l.r.Peek(0) == '=' {
l.r.Move(1)
return GtGtEqToken
}
return GtGtToken
}
return opTokens[c]
}
func (l *Lexer) consumeIdentifierToken() bool {
c := l.r.Peek(0)
if identifierStartTable[c] {
l.r.Move(1)
} else if 0xC0 <= c {
if r, n := l.r.PeekRune(0); unicode.IsOneOf(identifierStart, r) {
l.r.Move(n)
} else {
return false
}
} else if !l.consumeUnicodeEscape() {
return false
}
for {
c := l.r.Peek(0)
if identifierTable[c] {
l.r.Move(1)
} else if 0xC0 <= c {
if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
l.r.Move(n)
} else {
break
}
} else {
break
}
}
return true
}
func (l *Lexer) consumeNumericToken() TokenType {
// assume to be on 0 1 2 3 4 5 6 7 8 9 .
first := l.r.Peek(0)
if first == '0' {
l.r.Move(1)
if l.r.Peek(0) == 'x' || l.r.Peek(0) == 'X' {
l.r.Move(1)
if l.consumeHexDigit() {
for l.consumeHexDigit() {
}
return HexadecimalToken
}
l.err = parse.NewErrorLexer(l.r, "invalid hexadecimal number")
return ErrorToken
} else if l.r.Peek(0) == 'b' || l.r.Peek(0) == 'B' {
l.r.Move(1)
if l.consumeBinaryDigit() {
for l.consumeBinaryDigit() {
}
return BinaryToken
}
l.err = parse.NewErrorLexer(l.r, "invalid binary number")
return ErrorToken
} else if l.r.Peek(0) == 'o' || l.r.Peek(0) == 'O' {
l.r.Move(1)
if l.consumeOctalDigit() {
for l.consumeOctalDigit() {
}
return OctalToken
}
l.err = parse.NewErrorLexer(l.r, "invalid octal number")
return ErrorToken
} else if l.r.Peek(0) == 'n' {
l.r.Move(1)
return BigIntToken
} else if '0' <= l.r.Peek(0) && l.r.Peek(0) <= '9' {
l.err = parse.NewErrorLexer(l.r, "legacy octal numbers are not supported")
return ErrorToken
}
} else if first != '.' {
for l.consumeDigit() {
}
}
// we have parsed a 0 or an integer number
c := l.r.Peek(0)
if c == '.' {
l.r.Move(1)
if l.consumeDigit() {
for l.consumeDigit() {
}
c = l.r.Peek(0)
} else if first == '.' {
// number starts with a dot and must be followed by digits
l.r.Move(-1)
return ErrorToken // may be dot or ellipsis
} else {
c = l.r.Peek(0)
}
} else if c == 'n' {
l.r.Move(1)
return BigIntToken
}
if c == 'e' || c == 'E' {
l.r.Move(1)
c = l.r.Peek(0)
if c == '+' || c == '-' {
l.r.Move(1)
}
if !l.consumeDigit() {
l.err = parse.NewErrorLexer(l.r, "invalid number")
return ErrorToken
}
for l.consumeDigit() {
}
}
return DecimalToken
}
func (l *Lexer) consumeStringToken() bool {
// assume to be on ' or "
mark := l.r.Pos()
delim := l.r.Peek(0)
l.r.Move(1)
for {
c := l.r.Peek(0)
if c == delim {
l.r.Move(1)
break
} else if c == '\\' {
l.r.Move(1)
if !l.consumeLineTerminator() {
if c := l.r.Peek(0); c == delim || c == '\\' {
l.r.Move(1)
}
}
continue
} else if c == '\n' || c == '\r' || c == 0 && l.r.Err() != nil {
l.r.Rewind(mark)
return false
}
l.r.Move(1)
}
return true
}
func (l *Lexer) consumeRegExpToken() bool {
// assume to be on /
l.r.Move(1)
inClass := false
for {
c := l.r.Peek(0)
if !inClass && c == '/' {
l.r.Move(1)
break
} else if c == '[' {
inClass = true
} else if c == ']' {
inClass = false
} else if c == '\\' {
l.r.Move(1)
if l.isLineTerminator() || l.r.Peek(0) == 0 && l.r.Err() != nil {
return false
}
} else if l.isLineTerminator() || c == 0 && l.r.Err() != nil {
return false
}
l.r.Move(1)
}
// flags
for {
c := l.r.Peek(0)
if identifierTable[c] {
l.r.Move(1)
} else if 0xC0 <= c {
if r, n := l.r.PeekRune(0); r == '\u200C' || r == '\u200D' || unicode.IsOneOf(identifierContinue, r) {
l.r.Move(n)
} else {
break
}
} else {
break
}
}
return true
}
func (l *Lexer) consumeTemplateToken() TokenType {
// assume to be on ` or } when already within template
continuation := l.r.Peek(0) == '}'
l.r.Move(1)
for {
c := l.r.Peek(0)
if c == '`' {
l.templateLevels = l.templateLevels[:len(l.templateLevels)-1]
l.r.Move(1)
if continuation {
return TemplateEndToken
}
return TemplateToken
} else if c == '$' && l.r.Peek(1) == '{' {
l.level++
l.r.Move(2)
if continuation {
return TemplateMiddleToken
}
return TemplateStartToken
} else if c == '\\' {
l.r.Move(1)
if c := l.r.Peek(0); c != 0 {
l.r.Move(1)
}
continue
} else if c == 0 && l.r.Err() != nil {
if continuation {
return TemplateEndToken
}
return TemplateToken
}
l.r.Move(1)
}
}
var identifierStartTable = [256]bool{
// ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, true, false, false, false, // $
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
true, true, true, false, false, false, false, true, // X, Y, Z, _
false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
true, true, true, false, false, false, false, false, // x, y, z
// non-ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
}
var identifierTable = [256]bool{
// ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, true, false, false, false, // $
false, false, false, false, false, false, false, false,
true, true, true, true, true, true, true, true, // 0, 1, 2, 3, 4, 5, 6, 7
true, true, false, false, false, false, false, false, // 8, 9
false, true, true, true, true, true, true, true, // A, B, C, D, E, F, G
true, true, true, true, true, true, true, true, // H, I, J, K, L, M, N, O
true, true, true, true, true, true, true, true, // P, Q, R, S, T, U, V, W
true, true, true, false, false, false, false, true, // X, Y, Z, _
false, true, true, true, true, true, true, true, // a, b, c, d, e, f, g
true, true, true, true, true, true, true, true, // h, i, j, k, l, m, n, o
true, true, true, true, true, true, true, true, // p, q, r, s, t, u, v, w
true, true, true, false, false, false, false, false, // x, y, z
// non-ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
}
parse-2.5.21/js/lex_test.go 0000664 0000000 0000000 00000027440 14117006534 0015515 0 ustar 00root root 0000000 0000000 package js
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type TTs []TokenType
func TestTokens(t *testing.T) {
var tokenTests = []struct {
js string
expected []TokenType
}{
{" \t\v\f\u00A0\uFEFF\u2000", TTs{}}, // WhitespaceToken
{"\n\r\r\n\u2028\u2029", TTs{LineTerminatorToken}},
{"5.2 .04 1. 2.e3 0x0F 5e99", TTs{DecimalToken, DecimalToken, DecimalToken, DecimalToken, HexadecimalToken, DecimalToken}},
{"0o22 0b11", TTs{OctalToken, BinaryToken}},
{"0n 2345n 435.333n", TTs{BigIntToken, BigIntToken, DecimalToken, ErrorToken}},
{"a = 'string'", TTs{IdentifierToken, EqToken, StringToken}},
{"/*comment*/ //comment", TTs{CommentToken, CommentToken}},
{"{ } ( ) [ ]", TTs{OpenBraceToken, CloseBraceToken, OpenParenToken, CloseParenToken, OpenBracketToken, CloseBracketToken}},
{". ; , < > <= ...", TTs{DotToken, SemicolonToken, CommaToken, LtToken, GtToken, LtEqToken, EllipsisToken}},
{">= == != === !==", TTs{GtEqToken, EqEqToken, NotEqToken, EqEqEqToken, NotEqEqToken}},
{"+ - * / % ** ++ --", TTs{AddToken, SubToken, MulToken, DivToken, ModToken, ExpToken, IncrToken, DecrToken}},
{"<< >> >>> & | ^", TTs{LtLtToken, GtGtToken, GtGtGtToken, BitAndToken, BitOrToken, BitXorToken}},
{"! ~ && || ? : ?? ?.", TTs{NotToken, BitNotToken, AndToken, OrToken, QuestionToken, ColonToken, NullishToken, OptChainToken}},
{"= += -= *= **= /= %= <<=", TTs{EqToken, AddEqToken, SubEqToken, MulEqToken, ExpEqToken, DivEqToken, ModEqToken, LtLtEqToken}},
{">>= >>>= &= |= ^= =>", TTs{GtGtEqToken, GtGtGtEqToken, BitAndEqToken, BitOrEqToken, BitXorEqToken, ArrowToken}},
{"&&= ||= ??=", TTs{AndEqToken, OrEqToken, NullishEqToken}},
{"?.5", TTs{QuestionToken, DecimalToken}},
{"?.a", TTs{OptChainToken, IdentifierToken}},
{"await break case catch class const continue", TTs{AwaitToken, BreakToken, CaseToken, CatchToken, ClassToken, ConstToken, ContinueToken}},
{"debugger default delete do else enum export extends", TTs{DebuggerToken, DefaultToken, DeleteToken, DoToken, ElseToken, EnumToken, ExportToken, ExtendsToken}},
{"false finally for function if import in instanceof", TTs{FalseToken, FinallyToken, ForToken, FunctionToken, IfToken, ImportToken, InToken, InstanceofToken}},
{"new null return super switch this throw true", TTs{NewToken, NullToken, ReturnToken, SuperToken, SwitchToken, ThisToken, ThrowToken, TrueToken}},
{"try typeof var void while with yield", TTs{TryToken, TypeofToken, VarToken, VoidToken, WhileToken, WithToken, YieldToken}},
{"implements interface let package private protected public static", TTs{ImplementsToken, InterfaceToken, LetToken, PackageToken, PrivateToken, ProtectedToken, PublicToken, StaticToken}},
{"as async from get meta of set target", TTs{AsToken, AsyncToken, FromToken, GetToken, MetaToken, OfToken, SetToken, TargetToken}},
{"/*co\nm\u2028m/*ent*/ //co//mment\u2029//comment", TTs{CommentLineTerminatorToken, CommentToken, LineTerminatorToken, CommentToken}},
{"<!-", TTs{LtToken, NotToken, SubToken}},
{"1<!--2\n", TTs{DecimalToken, CommentToken, LineTerminatorToken}},
{"x=y-->10\n", TTs{IdentifierToken, EqToken, IdentifierToken, DecrToken, GtToken, DecimalToken, LineTerminatorToken}},
{" /*comment*/ -->nothing\n", TTs{CommentToken, DecrToken, GtToken, IdentifierToken, LineTerminatorToken}},
{"1 /*comment\nmultiline*/ -->nothing\n", TTs{DecimalToken, CommentLineTerminatorToken, CommentToken, LineTerminatorToken}},
{"$ _\u200C \\u2000 \u200C", TTs{IdentifierToken, IdentifierToken, IdentifierToken, ErrorToken}},
{">>>=>>>>=", TTs{GtGtGtEqToken, GtGtGtToken, GtEqToken}},
{"1/", TTs{DecimalToken, DivToken}},
{"1/=", TTs{DecimalToken, DivEqToken}},
{"'str\\i\\'ng'", TTs{StringToken}},
{"'str\\\\'abc", TTs{StringToken, IdentifierToken}},
{"'str\\\ni\\\\u00A0ng'", TTs{StringToken}},
{"'str\u2028\u2029ing'", TTs{StringToken}},
{"0b0101 0o0707 0b17", TTs{BinaryToken, OctalToken, BinaryToken, DecimalToken}},
{"`template`", TTs{TemplateToken}},
{"`a${x+y}b`", TTs{TemplateStartToken, IdentifierToken, AddToken, IdentifierToken, TemplateEndToken}},
{"`tmpl${x}tmpl${x}`", TTs{TemplateStartToken, IdentifierToken, TemplateMiddleToken, IdentifierToken, TemplateEndToken}},
{"`temp\nlate`", TTs{TemplateToken}},
{"`outer${{x: 10}}bar${ raw`nested${2}endnest` }end`", TTs{TemplateStartToken, OpenBraceToken, IdentifierToken, ColonToken, DecimalToken, CloseBraceToken, TemplateMiddleToken, IdentifierToken, TemplateStartToken, DecimalToken, TemplateEndToken, TemplateEndToken}},
{"`tmpl ${ a ? '' : `tmpl2 ${b ? 'b' : 'c'}` }`", TTs{TemplateStartToken, IdentifierToken, QuestionToken, StringToken, ColonToken, TemplateStartToken, IdentifierToken, QuestionToken, StringToken, ColonToken, StringToken, TemplateEndToken, TemplateEndToken}},
// early endings
{"'string", TTs{ErrorToken}},
{"'\n", TTs{ErrorToken}},
{"'\u2028", TTs{ErrorToken}},
{"'str\\\U00100000ing\\0'", TTs{StringToken}},
{"'strin\\00g'", TTs{StringToken}},
{"/*comment", TTs{CommentToken}},
{"a=/regexp", TTs{IdentifierToken, EqToken, DivToken, IdentifierToken}},
{"\\u002", TTs{ErrorToken}},
{"`template", TTs{TemplateToken}},
{"`template${x}template", TTs{TemplateStartToken, IdentifierToken, TemplateEndToken}},
{"a++=1", TTs{IdentifierToken, IncrToken, EqToken, DecimalToken}},
{"a++==1", TTs{IdentifierToken, IncrToken, EqEqToken, DecimalToken}},
{"a++===1", TTs{IdentifierToken, IncrToken, EqEqEqToken, DecimalToken}},
// null characters
{"'string\x00'return", TTs{StringToken, ReturnToken}},
{"//comment\x00comment\nreturn", TTs{CommentToken, LineTerminatorToken, ReturnToken}},
{"/*comment\x00*/return", TTs{CommentToken, ReturnToken}},
{"`template\x00`return", TTs{TemplateToken, ReturnToken}},
{"`template\\\x00`return", TTs{TemplateToken, ReturnToken}},
// numbers
{"0xg", TTs{ErrorToken}},
{"0.f", TTs{DecimalToken, ErrorToken}},
{"0bg", TTs{ErrorToken}},
{"0og", TTs{ErrorToken}},
{"010", TTs{ErrorToken}}, // Decimal(0) Decimal(10) Identifier(xF)
{"50e+-0", TTs{ErrorToken}},
{"5.a", TTs{DecimalToken, ErrorToken}},
{"5..a", TTs{DecimalToken, DotToken, IdentifierToken}},
// coverage
{"Ø a〉", TTs{IdentifierToken, IdentifierToken, ErrorToken}},
{"\u00A0\uFEFF\u2000", TTs{}},
{"\u2028\u2029", TTs{LineTerminatorToken}},
{"\\u0029ident", TTs{IdentifierToken}},
{"\\u{0029FEF}ident", TTs{IdentifierToken}},
{"\\u{}", TTs{ErrorToken}},
{"\\ugident", TTs{ErrorToken}},
{"'str\ring'", TTs{ErrorToken}},
{"a=/\\\n", TTs{IdentifierToken, EqToken, DivToken, ErrorToken}},
{"a=/x\n", TTs{IdentifierToken, EqToken, DivToken, IdentifierToken, LineTerminatorToken}},
{"`\\``", TTs{TemplateToken}},
{"`\\${ 1 }`", TTs{TemplateToken}},
{"`\\\r\n`", TTs{TemplateToken}},
// go fuzz
{"`", TTs{TemplateToken}},
}
for _, tt := range tokenTests {
t.Run(tt.js, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.js))
i := 0
tokens := []TokenType{}
for {
token, _ := l.Next()
if token == ErrorToken {
if l.Err() != io.EOF {
tokens = append(tokens, token)
}
break
} else if token == WhitespaceToken {
continue
}
tokens = append(tokens, token)
i++
}
test.T(t, tokens, tt.expected, "token types must match")
})
}
// coverage
for _, start := range []int{0, 0x0100, 0x0200, 0x0600, 0x0800} {
for i := start; ; i++ {
if TokenType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
}
test.That(t, IsPunctuator(CommaToken))
test.That(t, IsPunctuator(GtGtEqToken))
test.That(t, !IsPunctuator(WhileToken))
test.That(t, !IsOperator(CommaToken))
test.That(t, IsOperator(GtGtEqToken))
test.That(t, !IsOperator(WhileToken))
test.That(t, !IsIdentifier(CommaToken))
test.That(t, !IsIdentifier(GtGtEqToken))
test.That(t, IsReservedWord(WhileToken))
test.That(t, IsIdentifier(AsyncToken))
test.That(t, IsIdentifierName(WhileToken))
test.That(t, IsIdentifierName(AsToken))
test.That(t, IsIdentifierStart([]byte("a")))
test.That(t, !IsIdentifierStart([]byte("6")))
test.That(t, !IsIdentifierStart([]byte("[")))
test.That(t, IsIdentifierContinue([]byte("a")))
test.That(t, IsIdentifierContinue([]byte("6")))
test.That(t, !IsIdentifierContinue([]byte("[")))
test.That(t, IsIdentifierEnd([]byte(".a")))
test.That(t, IsIdentifierEnd([]byte(".6")))
test.That(t, !IsIdentifierEnd([]byte(".[")))
}
func TestRegExp(t *testing.T) {
var tokenTests = []struct {
js string
expected []TokenType
}{
{"a = /[a-z/]/g", TTs{IdentifierToken, EqToken, RegExpToken}},
{"a=/=/g1", TTs{IdentifierToken, EqToken, RegExpToken}},
{"a = /'\\\\/\n", TTs{IdentifierToken, EqToken, RegExpToken, LineTerminatorToken}},
{"a=/\\//g1", TTs{IdentifierToken, EqToken, RegExpToken}},
{"new RegExp(a + /\\d{1,2}/.source)", TTs{NewToken, IdentifierToken, OpenParenToken, IdentifierToken, AddToken, RegExpToken, DotToken, IdentifierToken, CloseParenToken}},
{"a=/regexp\x00/;return", TTs{IdentifierToken, EqToken, RegExpToken, SemicolonToken, ReturnToken}},
{"a=/regexp\\\x00/;return", TTs{IdentifierToken, EqToken, RegExpToken, SemicolonToken, ReturnToken}},
{"a=/x/\u200C\u3009", TTs{IdentifierToken, EqToken, RegExpToken, ErrorToken}},
{"a=/end", TTs{IdentifierToken, EqToken, ErrorToken}},
{"a=/\\\nend", TTs{IdentifierToken, EqToken, ErrorToken}},
{"a=/\\\u2028", TTs{IdentifierToken, EqToken, ErrorToken}},
{"a=/regexp/Ø", TTs{IdentifierToken, EqToken, RegExpToken}},
}
for _, tt := range tokenTests {
t.Run(tt.js, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.js))
i := 0
tokens := []TokenType{}
for {
token, _ := l.Next()
if token == DivToken || token == DivEqToken {
token, _ = l.RegExp()
}
if token == ErrorToken {
if l.Err() != io.EOF {
tokens = append(tokens, token)
}
break
} else if token == WhitespaceToken {
continue
}
tokens = append(tokens, token)
i++
}
test.T(t, tokens, tt.expected, "token types must match")
})
}
token, _ := NewLexer(parse.NewInputString("")).RegExp()
test.T(t, token, ErrorToken)
}
func TestOffset(t *testing.T) {
z := parse.NewInputString(`var i=5;`)
l := NewLexer(z)
test.T(t, z.Offset(), 0)
_, _ = l.Next()
test.T(t, z.Offset(), 3) // var
_, _ = l.Next()
test.T(t, z.Offset(), 4) // ws
_, _ = l.Next()
test.T(t, z.Offset(), 5) // i
_, _ = l.Next()
test.T(t, z.Offset(), 6) // =
_, _ = l.Next()
test.T(t, z.Offset(), 7) // 5
_, _ = l.Next()
test.T(t, z.Offset(), 8) // ;
}
func TestLexerErrors(t *testing.T) {
l := NewLexer(parse.NewInputString("@"))
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected @")
l = NewLexer(parse.NewInputString("\x00"))
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected 0x00")
l = NewLexer(parse.NewInputString("\x7f"))
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected 0x7F")
l = NewLexer(parse.NewInputString("\u200F"))
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected U+200F")
l = NewLexer(parse.NewInputString("\u2010"))
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected \u2010")
l = NewLexer(parse.NewInputString(""))
l.RegExp()
test.T(t, l.Err().(*parse.Error).Message, "expected / or /=")
l = NewLexer(parse.NewInputString("/"))
l.Next()
l.RegExp()
test.T(t, l.Err().(*parse.Error).Message, "unexpected EOF or newline")
l = NewLexer(parse.NewInputString("5a"))
l.Next()
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "unexpected identifier after number")
l = NewLexer(parse.NewInputString(".0E"))
l.Next()
l.Next()
test.T(t, l.Err().(*parse.Error).Message, "invalid number")
}
////////////////////////////////////////////////////////////////
func ExampleNewLexer() {
l := NewLexer(parse.NewInputString("var x = 'lorem ipsum';"))
out := ""
for {
tt, data := l.Next()
if tt == ErrorToken {
break
}
out += string(data)
}
fmt.Println(out)
// Output: var x = 'lorem ipsum';
}
parse-2.5.21/js/parse.go 0000664 0000000 0000000 00000161602 14117006534 0014777 0 ustar 00root root 0000000 0000000 package js
import (
"errors"
"fmt"
"io"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/parse/v2/buffer"
)
// Parser is the state for the parser.
type Parser struct {
l *Lexer
err error
data []byte
tt TokenType
prevLT bool
inFor bool
async, generator bool
assumeArrowFunc bool
allowDirectivePrologue bool
stmtLevel int
exprLevel int
scope *Scope
}
// Parse returns a JS AST tree of.
func Parse(r *parse.Input) (*AST, error) {
ast := &AST{}
p := &Parser{
l: NewLexer(r),
tt: WhitespaceToken, // trick so that next() works
}
// process shebang
if r.Peek(0) == '#' && r.Peek(1) == '!' {
r.Move(2)
p.l.consumeSingleLineComment() // consume till end-of-line
ast.Comments = append(ast.Comments, r.Shift())
}
p.tt, p.data = p.l.Next()
for p.tt == CommentToken || p.tt == CommentLineTerminatorToken {
ast.Comments = append(ast.Comments, p.data)
p.tt, p.data = p.l.Next()
if p.tt == WhitespaceToken || p.tt == LineTerminatorToken {
p.tt, p.data = p.l.Next()
}
}
if p.tt == WhitespaceToken || p.tt == LineTerminatorToken {
p.next()
}
// prevLT may be wrong but that is not a problem
ast.BlockStmt = p.parseModule()
if p.err == nil {
p.err = p.l.Err()
} else {
offset := p.l.r.Offset() - len(p.data)
p.err = parse.NewError(buffer.NewReader(p.l.r.Bytes()), offset, p.err.Error())
}
if p.err == io.EOF {
p.err = nil
}
return ast, p.err
}
////////////////////////////////////////////////////////////////
func (p *Parser) next() {
p.prevLT = false
p.tt, p.data = p.l.Next()
for p.tt == WhitespaceToken || p.tt == LineTerminatorToken || p.tt == CommentToken || p.tt == CommentLineTerminatorToken {
if p.tt == LineTerminatorToken || p.tt == CommentLineTerminatorToken {
p.prevLT = true
}
p.tt, p.data = p.l.Next()
}
}
func (p *Parser) failMessage(msg string, args ...interface{}) {
if p.err == nil {
p.err = fmt.Errorf(msg, args...)
p.tt = ErrorToken
}
}
func (p *Parser) fail(in string, expected ...TokenType) {
if p.err == nil {
msg := "unexpected"
if 0 < len(expected) {
msg = "expected"
for i, tt := range expected[:len(expected)-1] {
if 0 < i {
msg += ","
}
msg += " " + tt.String() + ""
}
if 2 < len(expected) {
msg += ", or"
} else if 1 < len(expected) {
msg += " or"
}
msg += " " + expected[len(expected)-1].String() + " instead of"
}
if p.tt == ErrorToken {
if p.l.Err() == io.EOF {
msg += " EOF"
} else if lexerErr, ok := p.l.Err().(*parse.Error); ok {
msg = lexerErr.Message
} else {
// does not happen
}
} else {
msg += " " + string(p.data) + ""
}
if in != "" {
msg += " in " + in
}
p.err = errors.New(msg)
p.tt = ErrorToken
}
}
func (p *Parser) consume(in string, tt TokenType) bool {
if p.tt != tt {
p.fail(in, tt)
return false
}
p.next()
return true
}
// TODO: refactor
//type ScopeState struct {
// scope *Scope
// async bool
// generator bool
// assumeArrowFunc bool
//}
func (p *Parser) enterScope(scope *Scope, isFunc bool) *Scope {
// create a new scope object and add it to the parent
parent := p.scope
p.scope = scope
*scope = Scope{
Parent: parent,
Func: nil,
Declared: VarArray{},
Undeclared: VarArray{},
NumVarDecls: 0,
NumArguments: 0,
HasWith: false,
}
if isFunc {
scope.Func = scope
} else if parent != nil {
scope.Func = parent.Func
}
return parent
}
func (p *Parser) exitScope(parent *Scope) {
p.scope.HoistUndeclared()
p.scope = parent
}
func (p *Parser) parseModule() (module BlockStmt) {
p.enterScope(&module.Scope, true)
p.allowDirectivePrologue = true
for {
switch p.tt {
case ErrorToken:
return
case ImportToken:
p.next()
if p.tt == OpenParenToken {
// could be an import call expression
left := &LiteralExpr{ImportToken, []byte("import")}
p.exprLevel++
suffix := p.parseExpressionSuffix(left, OpExpr, OpCall)
p.exprLevel--
module.List = append(module.List, &ExprStmt{suffix})
} else {
importStmt := p.parseImportStmt()
module.List = append(module.List, &importStmt)
}
case ExportToken:
exportStmt := p.parseExportStmt()
module.List = append(module.List, &exportStmt)
default:
module.List = append(module.List, p.parseStmt(true))
}
}
}
func (p *Parser) parseStmt(allowDeclaration bool) (stmt IStmt) {
p.stmtLevel++
if 1000 < p.stmtLevel {
p.failMessage("too many nested statements")
return nil
}
switch tt := p.tt; tt {
case OpenBraceToken:
stmt = p.parseBlockStmt("block statement")
case ConstToken, VarToken:
if !allowDeclaration && tt == ConstToken {
p.fail("statement")
return
}
p.next()
varDecl := p.parseVarDecl(tt)
stmt = &varDecl
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
if tt == ConstToken {
p.fail("const declaration")
} else {
p.fail("var statement")
}
return
}
case LetToken:
let := p.data
p.next()
if allowDeclaration && (IsIdentifier(p.tt) || p.tt == YieldToken || p.tt == AwaitToken || p.tt == OpenBracketToken || p.tt == OpenBraceToken) {
varDecl := p.parseVarDecl(tt)
stmt = &varDecl
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
p.fail("let declaration")
return
}
} else {
// expression
stmt = &ExprStmt{p.parseIdentifierExpression(OpExpr, let)}
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
p.fail("expression")
return
}
}
case IfToken:
p.next()
if !p.consume("if statement", OpenParenToken) {
return
}
cond := p.parseExpression(OpExpr)
if !p.consume("if statement", CloseParenToken) {
return
}
body := p.parseStmt(false)
var elseBody IStmt
if p.tt == ElseToken {
p.next()
elseBody = p.parseStmt(false)
}
stmt = &IfStmt{cond, body, elseBody}
case ContinueToken, BreakToken:
tt := p.tt
p.next()
var label []byte
if !p.prevLT && p.isIdentifierReference(p.tt) {
label = p.data
p.next()
}
stmt = &BranchStmt{tt, label}
case ReturnToken:
p.next()
var value IExpr
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
value = p.parseExpression(OpExpr)
}
stmt = &ReturnStmt{value}
case WithToken:
p.next()
if !p.consume("with statement", OpenParenToken) {
return
}
cond := p.parseExpression(OpExpr)
if !p.consume("with statement", CloseParenToken) {
return
}
p.scope.Func.HasWith = true
stmt = &WithStmt{cond, p.parseStmt(false)}
case DoToken:
stmt = &DoWhileStmt{}
p.next()
body := p.parseStmt(false)
if !p.consume("do-while statement", WhileToken) {
return
}
if !p.consume("do-while statement", OpenParenToken) {
return
}
stmt = &DoWhileStmt{p.parseExpression(OpExpr), body}
if !p.consume("do-while statement", CloseParenToken) {
return
}
case WhileToken:
p.next()
if !p.consume("while statement", OpenParenToken) {
return
}
cond := p.parseExpression(OpExpr)
if !p.consume("while statement", CloseParenToken) {
return
}
stmt = &WhileStmt{cond, p.parseStmt(false)}
case ForToken:
p.next()
await := p.async && p.tt == AwaitToken
if await {
p.next()
}
if !p.consume("for statement", OpenParenToken) {
return
}
body := &BlockStmt{}
parent := p.enterScope(&body.Scope, false)
var init IExpr
p.inFor = true
if p.tt == VarToken || p.tt == LetToken || p.tt == ConstToken {
tt := p.tt
p.next()
varDecl := p.parseVarDecl(tt)
if p.tt != SemicolonToken && (1 < len(varDecl.List) || varDecl.List[0].Default != nil) {
p.fail("for statement")
return
} else if p.tt == SemicolonToken && varDecl.List[0].Default == nil {
// all but the first item were already verified
if _, ok := varDecl.List[0].Binding.(*Var); !ok {
p.fail("for statement")
return
}
}
init = &varDecl
} else if p.tt != SemicolonToken {
init = p.parseExpression(OpExpr)
}
p.inFor = false
if p.tt == SemicolonToken {
var cond, post IExpr
if await {
p.fail("for statement", OfToken)
return
}
p.next()
if p.tt != SemicolonToken {
cond = p.parseExpression(OpExpr)
}
if !p.consume("for statement", SemicolonToken) {
return
}
if p.tt != CloseParenToken {
post = p.parseExpression(OpExpr)
}
if !p.consume("for statement", CloseParenToken) {
return
}
p.scope.MarkForInit()
if p.tt == OpenBraceToken {
body.List = p.parseStmtList("")
} else if p.tt != SemicolonToken {
body.List = []IStmt{p.parseStmt(false)}
}
stmt = &ForStmt{init, cond, post, body}
} else if p.tt == InToken {
if await {
p.fail("for statement", OfToken)
return
}
p.next()
value := p.parseExpression(OpExpr)
if !p.consume("for statement", CloseParenToken) {
return
}
p.scope.MarkForInit()
if p.tt == OpenBraceToken {
body.List = p.parseStmtList("")
} else if p.tt != SemicolonToken {
body.List = []IStmt{p.parseStmt(false)}
}
stmt = &ForInStmt{init, value, body}
} else if p.tt == OfToken {
p.next()
value := p.parseExpression(OpAssign)
if !p.consume("for statement", CloseParenToken) {
return
}
p.scope.MarkForInit()
if p.tt == OpenBraceToken {
body.List = p.parseStmtList("")
} else if p.tt != SemicolonToken {
body.List = []IStmt{p.parseStmt(false)}
}
stmt = &ForOfStmt{await, init, value, body}
} else {
p.fail("for statement", InToken, OfToken, SemicolonToken)
return
}
p.exitScope(parent)
case SwitchToken:
p.next()
if !p.consume("switch statement", OpenParenToken) {
return
}
init := p.parseExpression(OpExpr)
if !p.consume("switch statement", CloseParenToken) {
return
}
// case block
if !p.consume("switch statement", OpenBraceToken) {
return
}
switchStmt := &SwitchStmt{Init: init}
parent := p.enterScope(&switchStmt.Scope, false)
for {
if p.tt == ErrorToken {
p.fail("switch statement")
return
} else if p.tt == CloseBraceToken {
p.next()
break
}
clause := p.tt
var list IExpr
if p.tt == CaseToken {
p.next()
list = p.parseExpression(OpExpr)
} else if p.tt == DefaultToken {
p.next()
} else {
p.fail("switch statement", CaseToken, DefaultToken)
return
}
if !p.consume("switch statement", ColonToken) {
return
}
var stmts []IStmt
for p.tt != CaseToken && p.tt != DefaultToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
stmts = append(stmts, p.parseStmt(true))
}
switchStmt.List = append(switchStmt.List, CaseClause{clause, list, stmts})
}
p.exitScope(parent)
stmt = switchStmt
case FunctionToken:
if !allowDeclaration {
p.fail("statement")
return
}
stmt = p.parseFuncDecl()
case AsyncToken: // async function
if !allowDeclaration {
p.fail("statement")
return
}
async := p.data
p.next()
if p.tt == FunctionToken && !p.prevLT {
stmt = p.parseAsyncFuncDecl()
} else {
// expression
stmt = &ExprStmt{p.parseAsyncExpression(OpExpr, async)}
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
p.fail("expression")
return
}
}
case ClassToken:
if !allowDeclaration {
p.fail("statement")
return
}
stmt = p.parseClassDecl()
case ThrowToken:
p.next()
var value IExpr
if !p.prevLT {
value = p.parseExpression(OpExpr)
}
stmt = &ThrowStmt{value}
case TryToken:
p.next()
body := p.parseBlockStmt("try statement")
var binding IBinding
var catch, finally *BlockStmt
if p.tt == CatchToken {
p.next()
catch = &BlockStmt{}
parent := p.enterScope(&catch.Scope, false)
if p.tt == OpenParenToken {
p.next()
binding = p.parseBinding(CatchDecl) // local to block scope of catch
if !p.consume("try-catch statement", CloseParenToken) {
return
}
}
catch.List = p.parseStmtList("try-catch statement")
p.exitScope(parent)
} else if p.tt != FinallyToken {
p.fail("try statement", CatchToken, FinallyToken)
return
}
if p.tt == FinallyToken {
p.next()
finally = p.parseBlockStmt("try-finally statement")
}
stmt = &TryStmt{body, binding, catch, finally}
case DebuggerToken:
p.next()
stmt = &DebuggerStmt{}
case SemicolonToken, ErrorToken:
stmt = &EmptyStmt{}
default:
if p.isIdentifierReference(p.tt) {
// labelled statement or expression
label := p.data
p.next()
if p.tt == ColonToken {
p.next()
stmt = &LabelledStmt{label, p.parseStmt(true)} // allows illegal async function, generator function, let, const, or class declarations
} else {
// expression
stmt = &ExprStmt{p.parseIdentifierExpression(OpExpr, label)}
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
p.fail("expression")
return
}
}
} else {
// expression
stmt = &ExprStmt{p.parseExpression(OpExpr)}
if !p.prevLT && p.tt != SemicolonToken && p.tt != CloseBraceToken && p.tt != ErrorToken {
p.fail("expression")
return
}
if p.allowDirectivePrologue {
if lit, ok := stmt.(*ExprStmt).Value.(*LiteralExpr); ok && lit.TokenType == StringToken {
stmt = &DirectivePrologueStmt{lit.Data}
} else {
p.allowDirectivePrologue = false
}
}
}
}
if p.tt == SemicolonToken {
p.next()
}
p.stmtLevel--
return
}
func (p *Parser) parseStmtList(in string) (list []IStmt) {
if !p.consume(in, OpenBraceToken) {
return
}
for {
if p.tt == ErrorToken {
p.fail("")
return
} else if p.tt == CloseBraceToken {
p.next()
break
}
list = append(list, p.parseStmt(true))
}
return
}
func (p *Parser) parseBlockStmt(in string) (blockStmt *BlockStmt) {
blockStmt = &BlockStmt{}
parent := p.enterScope(&blockStmt.Scope, false)
blockStmt.List = p.parseStmtList(in)
p.exitScope(parent)
return
}
func (p *Parser) parseImportStmt() (importStmt ImportStmt) {
// assume we're passed import
if p.tt == StringToken {
importStmt.Module = p.data
p.next()
} else {
if IsIdentifier(p.tt) || p.tt == YieldToken || p.tt == AwaitToken {
importStmt.Default = p.data
p.next()
if p.tt == CommaToken {
p.next()
}
}
if p.tt == MulToken {
star := p.data
p.next()
if !p.consume("import statement", AsToken) {
return
}
if !IsIdentifier(p.tt) && p.tt != YieldToken && p.tt != AwaitToken {
p.fail("import statement", IdentifierToken)
return
}
importStmt.List = []Alias{Alias{star, p.data}}
p.next()
} else if p.tt == OpenBraceToken {
p.next()
for IsIdentifierName(p.tt) {
var name, binding []byte = nil, p.data
p.next()
if p.tt == AsToken {
p.next()
if !IsIdentifier(p.tt) && p.tt != YieldToken && p.tt != AwaitToken {
p.fail("import statement", IdentifierToken)
return
}
name = binding
binding = p.data
p.next()
}
importStmt.List = append(importStmt.List, Alias{name, binding})
if p.tt == CommaToken {
p.next()
if p.tt == CloseBraceToken {
importStmt.List = append(importStmt.List, Alias{})
break
}
}
}
if !p.consume("import statement", CloseBraceToken) {
return
}
}
if importStmt.Default == nil && len(importStmt.List) == 0 {
p.fail("import statement", StringToken, IdentifierToken, MulToken, OpenBraceToken)
return
}
if !p.consume("import statement", FromToken) {
return
}
if p.tt != StringToken {
p.fail("import statement", StringToken)
return
}
importStmt.Module = p.data
p.next()
}
if p.tt == SemicolonToken {
p.next()
}
return
}
func (p *Parser) parseExportStmt() (exportStmt ExportStmt) {
// assume we're at export
p.next()
if p.tt == MulToken || p.tt == OpenBraceToken {
if p.tt == MulToken {
star := p.data
p.next()
if p.tt == AsToken {
p.next()
if !IsIdentifierName(p.tt) {
p.fail("export statement", IdentifierToken)
return
}
exportStmt.List = []Alias{Alias{star, p.data}}
p.next()
} else {
exportStmt.List = []Alias{Alias{nil, star}}
}
if p.tt != FromToken {
p.fail("export statement", FromToken)
return
}
} else {
p.next()
for IsIdentifierName(p.tt) {
var name, binding []byte = nil, p.data
p.next()
if p.tt == AsToken {
p.next()
if !IsIdentifierName(p.tt) {
p.fail("export statement", IdentifierToken)
return
}
name = binding
binding = p.data
p.next()
}
exportStmt.List = append(exportStmt.List, Alias{name, binding})
if p.tt == CommaToken {
p.next()
if p.tt == CloseBraceToken {
exportStmt.List = append(exportStmt.List, Alias{})
break
}
}
}
if !p.consume("export statement", CloseBraceToken) {
return
}
}
if p.tt == FromToken {
p.next()
if p.tt != StringToken {
p.fail("export statement", StringToken)
return
}
exportStmt.Module = p.data
p.next()
}
} else if p.tt == VarToken || p.tt == ConstToken || p.tt == LetToken {
tt := p.tt
p.next()
varDecl := p.parseVarDecl(tt)
exportStmt.Decl = &varDecl
} else if p.tt == FunctionToken {
exportStmt.Decl = p.parseFuncDecl()
} else if p.tt == AsyncToken { // async function
p.next()
if p.tt != FunctionToken || p.prevLT {
p.fail("export statement", FunctionToken)
return
}
exportStmt.Decl = p.parseAsyncFuncDecl()
} else if p.tt == ClassToken {
exportStmt.Decl = p.parseClassDecl()
} else if p.tt == DefaultToken {
exportStmt.Default = true
p.next()
if p.tt == FunctionToken {
exportStmt.Decl = p.parseFuncExpr()
} else if p.tt == AsyncToken { // async function or async arrow function
async := p.data
p.next()
if p.tt == FunctionToken && !p.prevLT {
exportStmt.Decl = p.parseAsyncFuncExpr()
} else {
// expression
exportStmt.Decl = p.parseAsyncExpression(OpExpr, async)
}
} else if p.tt == ClassToken {
exportStmt.Decl = p.parseClassExpr()
} else {
exportStmt.Decl = p.parseExpression(OpAssign)
}
} else {
p.fail("export statement", MulToken, OpenBraceToken, VarToken, LetToken, ConstToken, FunctionToken, AsyncToken, ClassToken, DefaultToken)
return
}
if p.tt == SemicolonToken {
p.next()
}
return
}
func (p *Parser) parseVarDecl(tt TokenType) (varDecl VarDecl) {
// assume we're past var, let or const
varDecl.TokenType = tt
declType := LexicalDecl
if tt == VarToken {
declType = VariableDecl
p.scope.Func.NumVarDecls++
}
for {
// binding element, var declaration in for-in or for-of can never have a default
var bindingElement BindingElement
parentInFor := p.inFor
p.inFor = false
bindingElement.Binding = p.parseBinding(declType)
p.inFor = parentInFor
if p.tt == EqToken {
p.next()
bindingElement.Default = p.parseExpression(OpAssign)
} else if _, ok := bindingElement.Binding.(*Var); !ok && (!p.inFor || 0 < len(varDecl.List)) {
p.fail("var statement", EqToken)
return
}
varDecl.List = append(varDecl.List, bindingElement)
if p.tt == CommaToken {
p.next()
} else {
break
}
}
return
}
func (p *Parser) parseFuncParams(in string) (params Params) {
if !p.consume(in, OpenParenToken) {
return
}
for p.tt != CloseParenToken && p.tt != ErrorToken {
if p.tt == EllipsisToken {
// binding rest element
p.next()
params.Rest = p.parseBinding(ArgumentDecl)
p.consume(in, CloseParenToken)
return
}
params.List = append(params.List, p.parseBindingElement(ArgumentDecl))
if p.tt != CommaToken {
break
}
p.next()
}
if p.tt != CloseParenToken {
p.fail(in)
return
}
p.next()
// mark undeclared vars as arguments in `function f(a=b){var b}` where the b's are different vars
p.scope.MarkArguments()
return
}
func (p *Parser) parseFuncDecl() (funcDecl *FuncDecl) {
return p.parseAnyFunc(false, false)
}
func (p *Parser) parseAsyncFuncDecl() (funcDecl *FuncDecl) {
return p.parseAnyFunc(true, false)
}
func (p *Parser) parseFuncExpr() (funcDecl *FuncDecl) {
return p.parseAnyFunc(false, true)
}
func (p *Parser) parseAsyncFuncExpr() (funcDecl *FuncDecl) {
return p.parseAnyFunc(true, true)
}
func (p *Parser) parseAnyFunc(async, inExpr bool) (funcDecl *FuncDecl) {
// assume we're at function
p.next()
funcDecl = &FuncDecl{}
funcDecl.Async = async
funcDecl.Generator = p.tt == MulToken
if funcDecl.Generator {
p.next()
}
var ok bool
var name []byte
if inExpr && (IsIdentifier(p.tt) || p.tt == YieldToken || p.tt == AwaitToken) || !inExpr && p.isIdentifierReference(p.tt) {
name = p.data
if !inExpr {
funcDecl.Name, ok = p.scope.Declare(FunctionDecl, p.data)
if !ok {
p.failMessage("identifier %s has already been declared", string(p.data))
return
}
}
p.next()
} else if !inExpr {
p.fail("function declaration", IdentifierToken)
return
} else if p.tt != OpenParenToken {
p.fail("function declaration", IdentifierToken, OpenParenToken)
return
}
parent := p.enterScope(&funcDecl.Body.Scope, true)
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = funcDecl.Async, funcDecl.Generator
if inExpr && name != nil {
funcDecl.Name, _ = p.scope.Declare(ExprDecl, name) // cannot fail
}
funcDecl.Params = p.parseFuncParams("function declaration")
p.allowDirectivePrologue = true
funcDecl.Body.List = p.parseStmtList("function declaration")
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
return
}
func (p *Parser) parseClassDecl() (classDecl *ClassDecl) {
return p.parseAnyClass(false)
}
func (p *Parser) parseClassExpr() (classDecl *ClassDecl) {
return p.parseAnyClass(true)
}
func (p *Parser) parseAnyClass(inExpr bool) (classDecl *ClassDecl) {
// assume we're at class
p.next()
classDecl = &ClassDecl{}
if IsIdentifier(p.tt) || p.tt == YieldToken || p.tt == AwaitToken {
if !inExpr {
var ok bool
classDecl.Name, ok = p.scope.Declare(LexicalDecl, p.data)
if !ok {
p.failMessage("identifier %s has already been declared", string(p.data))
return
}
} else {
//classDecl.Name, ok = p.scope.Declare(ExprDecl, p.data) // classes do not register vars
classDecl.Name = &Var{p.data, nil, 1, ExprDecl}
}
p.next()
} else if !inExpr {
p.fail("class declaration", IdentifierToken)
return
}
if p.tt == ExtendsToken {
p.next()
classDecl.Extends = p.parseExpression(OpLHS)
}
if !p.consume("class declaration", OpenBraceToken) {
return
}
for {
if p.tt == ErrorToken {
p.fail("class declaration")
return
} else if p.tt == SemicolonToken {
p.next()
continue
} else if p.tt == CloseBraceToken {
p.next()
break
}
method, definition := p.parseClassElement()
if method != nil {
classDecl.Methods = append(classDecl.Methods, method)
} else {
classDecl.Definitions = append(classDecl.Definitions, definition)
}
}
return
}
func (p *Parser) parseClassElement() (method *MethodDecl, definition FieldDefinition) {
method = &MethodDecl{}
var data []byte
if p.tt == StaticToken {
method.Static = true
data = p.data
p.next()
}
if p.tt == MulToken {
method.Generator = true
p.next()
} else if p.tt == AsyncToken {
data = p.data
p.next()
if !p.prevLT {
method.Async = true
if p.tt == MulToken {
method.Generator = true
data = nil
p.next()
}
}
} else if p.tt == GetToken {
method.Get = true
data = p.data
p.next()
} else if p.tt == SetToken {
method.Set = true
data = p.data
p.next()
}
isFieldDefinition := false
if data != nil && p.tt == OpenParenToken {
method.Name.Literal = LiteralExpr{IdentifierToken, data}
if method.Async || method.Get || method.Set {
method.Async = false
method.Get = false
method.Set = false
} else {
method.Static = false
}
} else if data != nil && (p.tt == EqToken || p.tt == SemicolonToken || p.tt == CloseBraceToken) {
method.Name.Literal = LiteralExpr{IdentifierToken, data}
isFieldDefinition = true
} else if data == nil && p.tt == PrivateIdentifierToken {
method.Name.Literal = LiteralExpr{p.tt, p.data}
p.next()
isFieldDefinition = true
} else {
method.Name = p.parsePropertyName("method definition")
if data == nil && p.tt != OpenParenToken {
isFieldDefinition = true
}
}
if isFieldDefinition {
// FieldDefinition
definition.Name = method.Name
if p.tt == EqToken {
p.next()
definition.Init = p.parseExpression(OpAssign)
}
method = nil
return
}
parent := p.enterScope(&method.Body.Scope, true)
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = method.Async, method.Generator
method.Params = p.parseFuncParams("method definition")
p.allowDirectivePrologue = true
method.Body.List = p.parseStmtList("method definition")
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
return
}
func (p *Parser) parsePropertyName(in string) (propertyName PropertyName) {
if IsIdentifierName(p.tt) {
propertyName.Literal = LiteralExpr{IdentifierToken, p.data}
p.next()
} else if p.tt == StringToken {
// reinterpret string as identifier or number if we can, except for empty strings
if isIdent := AsIdentifierName(p.data[1 : len(p.data)-1]); isIdent {
propertyName.Literal = LiteralExpr{IdentifierToken, p.data[1 : len(p.data)-1]}
} else if isNum := AsDecimalLiteral(p.data[1 : len(p.data)-1]); isNum {
propertyName.Literal = LiteralExpr{DecimalToken, p.data[1 : len(p.data)-1]}
} else {
propertyName.Literal = LiteralExpr{p.tt, p.data}
}
p.next()
} else if IsNumeric(p.tt) {
propertyName.Literal = LiteralExpr{p.tt, p.data}
p.next()
} else if p.tt == OpenBracketToken {
p.next()
propertyName.Computed = p.parseExpression(OpAssign)
if !p.consume(in, CloseBracketToken) {
return
}
} else {
p.fail(in, IdentifierToken, StringToken, NumericToken, OpenBracketToken)
return
}
return
}
func (p *Parser) parseBindingElement(decl DeclType) (bindingElement BindingElement) {
// binding element
bindingElement.Binding = p.parseBinding(decl)
if p.tt == EqToken {
p.next()
bindingElement.Default = p.parseExpression(OpAssign)
}
return
}
func (p *Parser) parseBinding(decl DeclType) (binding IBinding) {
// binding identifier or binding pattern
if IsIdentifier(p.tt) || !p.generator && p.tt == YieldToken || !p.async && p.tt == AwaitToken {
var ok bool
binding, ok = p.scope.Declare(decl, p.data)
if !ok {
p.failMessage("identifier %s has already been declared", string(p.data))
return
}
p.next()
} else if p.tt == OpenBracketToken {
p.next()
array := BindingArray{}
if p.tt == CommaToken {
array.List = append(array.List, BindingElement{})
}
last := 0
for p.tt != CloseBracketToken {
// elision
for p.tt == CommaToken {
p.next()
if p.tt == CommaToken {
array.List = append(array.List, BindingElement{})
}
}
// binding rest element
if p.tt == EllipsisToken {
p.next()
array.Rest = p.parseBinding(decl)
if p.tt != CloseBracketToken {
p.fail("array binding pattern", CloseBracketToken)
return
}
break
} else if p.tt == CloseBracketToken {
array.List = array.List[:last]
break
}
array.List = append(array.List, p.parseBindingElement(decl))
last = len(array.List)
if p.tt != CommaToken && p.tt != CloseBracketToken {
p.fail("array binding pattern", CommaToken, CloseBracketToken)
return
}
}
p.next() // always CloseBracketToken
binding = &array
} else if p.tt == OpenBraceToken {
p.next()
object := BindingObject{}
for p.tt != CloseBraceToken {
// binding rest property
if p.tt == EllipsisToken {
p.next()
if !p.isIdentifierReference(p.tt) {
p.fail("object binding pattern", IdentifierToken)
return
}
var ok bool
object.Rest, ok = p.scope.Declare(decl, p.data)
if !ok {
p.failMessage("identifier %s has already been declared", string(p.data))
return
}
p.next()
if p.tt != CloseBraceToken {
p.fail("object binding pattern", CloseBraceToken)
return
}
break
}
item := BindingObjectItem{}
if p.isIdentifierReference(p.tt) {
name := p.data
item.Key = &PropertyName{LiteralExpr{IdentifierToken, p.data}, nil}
p.next()
if p.tt == ColonToken {
// property name + : + binding element
p.next()
item.Value = p.parseBindingElement(decl)
} else {
// single name binding
var ok bool
item.Key.Literal.Data = parse.Copy(item.Key.Literal.Data) // copy so that renaming doesn't rename the key
item.Value.Binding, ok = p.scope.Declare(decl, name)
if !ok {
p.failMessage("identifier %s has already been declared", string(name))
return
}
if p.tt == EqToken {
p.next()
item.Value.Default = p.parseExpression(OpAssign)
}
}
} else {
propertyName := p.parsePropertyName("object binding pattern")
item.Key = &propertyName
if !p.consume("object binding pattern", ColonToken) {
return
}
item.Value = p.parseBindingElement(decl)
}
object.List = append(object.List, item)
if p.tt == CommaToken {
p.next()
} else if p.tt != CloseBraceToken {
p.fail("object binding pattern", CommaToken, CloseBraceToken)
return
}
}
p.next() // always CloseBracketToken
binding = &object
} else {
p.fail("binding")
return
}
return
}
func (p *Parser) parseArrayLiteral() (array ArrayExpr) {
// assume we're on [
p.next()
prevComma := true
for {
if p.tt == ErrorToken {
p.fail("expression")
return
} else if p.tt == CloseBracketToken {
p.next()
break
} else if p.tt == CommaToken {
if prevComma {
array.List = append(array.List, Element{})
}
prevComma = true
p.next()
} else {
spread := p.tt == EllipsisToken
if spread {
p.next()
}
array.List = append(array.List, Element{p.parseAssignmentExpression(), spread})
prevComma = false
if spread && p.tt != CloseBracketToken {
p.assumeArrowFunc = false
}
}
}
return
}
func (p *Parser) parseObjectLiteral() (object ObjectExpr) {
// assume we're on {
p.next()
for {
if p.tt == ErrorToken {
p.fail("object literal", CloseBraceToken)
return
} else if p.tt == CloseBraceToken {
p.next()
break
}
property := Property{}
if p.tt == EllipsisToken {
p.next()
property.Spread = true
property.Value = p.parseAssignmentExpression()
if _, isIdent := property.Value.(*Var); !isIdent || p.tt != CloseBraceToken {
p.assumeArrowFunc = false
}
} else {
// try to parse as MethodDefinition, otherwise fall back to PropertyName:AssignExpr or IdentifierReference
var data []byte
method := MethodDecl{}
if p.tt == MulToken {
p.next()
method.Generator = true
} else if p.tt == AsyncToken {
data = p.data
p.next()
if !p.prevLT {
method.Async = true
if p.tt == MulToken {
p.next()
method.Generator = true
data = nil
}
} else {
method.Name.Literal = LiteralExpr{IdentifierToken, data}
data = nil
}
} else if p.tt == GetToken {
data = p.data
p.next()
method.Get = true
} else if p.tt == SetToken {
data = p.data
p.next()
method.Set = true
}
// PropertyName
if data != nil && !method.Generator && (p.tt == EqToken || p.tt == CommaToken || p.tt == CloseBraceToken || p.tt == ColonToken || p.tt == OpenParenToken) {
method.Name.Literal = LiteralExpr{IdentifierToken, data}
method.Async = false
method.Get = false
method.Set = false
} else if !method.Name.IsSet() { // did not parse async [LT]
method.Name = p.parsePropertyName("object literal")
if !method.Name.IsSet() {
return
}
}
if p.tt == OpenParenToken {
// MethodDefinition
parent := p.enterScope(&method.Body.Scope, true)
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = method.Async, method.Generator
method.Params = p.parseFuncParams("method definition")
method.Body.List = p.parseStmtList("method definition")
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
property.Value = &method
p.assumeArrowFunc = false
} else if p.tt == ColonToken {
// PropertyName : AssignmentExpression
p.next()
property.Name = &method.Name
property.Value = p.parseAssignmentExpression()
} else if method.Name.IsComputed() || !p.isIdentifierReference(method.Name.Literal.TokenType) {
p.fail("object literal", ColonToken, OpenParenToken)
return
} else {
// IdentifierReference (= AssignmentExpression)?
name := method.Name.Literal.Data
method.Name.Literal.Data = parse.Copy(method.Name.Literal.Data) // copy so that renaming doesn't rename the key
property.Name = &method.Name // set key explicitly so after renaming the original is still known
if p.assumeArrowFunc {
var ok bool
property.Value, ok = p.scope.Declare(ArgumentDecl, name)
if !ok {
property.Value = p.scope.Use(name)
p.assumeArrowFunc = false
}
} else {
property.Value = p.scope.Use(name)
}
if p.tt == EqToken {
p.next()
parentAssumeArrowFunc := p.assumeArrowFunc
p.assumeArrowFunc = false
property.Init = p.parseExpression(OpAssign)
p.assumeArrowFunc = parentAssumeArrowFunc
}
}
}
object.List = append(object.List, property)
if p.tt == CommaToken {
p.next()
} else if p.tt != CloseBraceToken {
p.fail("object literal")
return
}
}
return
}
func (p *Parser) parseTemplateLiteral(precLeft OpPrec) (template TemplateExpr) {
// assume we're on 'Template' or 'TemplateStart'
template.Prec = OpMember
if precLeft < OpMember {
template.Prec = OpCall
}
for p.tt == TemplateStartToken || p.tt == TemplateMiddleToken {
tpl := p.data
p.next()
template.List = append(template.List, TemplatePart{tpl, p.parseExpression(OpExpr)})
}
if p.tt != TemplateToken && p.tt != TemplateEndToken {
p.fail("template literal", TemplateToken)
return
}
template.Tail = p.data
p.next() // TemplateEndToken
return
}
func (p *Parser) parseArguments() (args Args) {
// assume we're on (
p.next()
args.List = make([]Arg, 0, 4)
for {
rest := p.tt == EllipsisToken
if rest {
p.next()
}
if p.tt == CloseParenToken || p.tt == ErrorToken {
break
}
args.List = append(args.List, Arg{
Value: p.parseExpression(OpAssign),
Rest: rest,
})
if p.tt == CommaToken {
p.next()
}
}
p.consume("arguments", CloseParenToken)
return
}
func (p *Parser) parseAsyncArrowFunc() (arrowFunc *ArrowFunc) {
// expect we're at Identifier or Yield or (
arrowFunc = &ArrowFunc{}
parent := p.enterScope(&arrowFunc.Body.Scope, true)
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = true, false
if IsIdentifier(p.tt) || !p.generator && p.tt == YieldToken {
ref, _ := p.scope.Declare(ArgumentDecl, p.data)
p.next()
arrowFunc.Params.List = []BindingElement{{Binding: ref}}
} else {
arrowFunc.Params = p.parseFuncParams("arrow function")
// could be CallExpression of: async(params)
if p.tt != ArrowToken {
}
}
arrowFunc.Async = true
arrowFunc.Body.List = p.parseArrowFuncBody()
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
return
}
func (p *Parser) parseIdentifierArrowFunc(v *Var) (arrowFunc *ArrowFunc) {
// expect we're at =>
arrowFunc = &ArrowFunc{}
parent := p.enterScope(&arrowFunc.Body.Scope, true)
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = false, false
if 1 < v.Uses {
v.Uses--
v, _ = p.scope.Declare(ArgumentDecl, v.Data) // cannot fail
} else {
// if v.Uses==1 it must be undeclared and be the last added
p.scope.Parent.Undeclared = p.scope.Parent.Undeclared[:len(p.scope.Parent.Undeclared)-1]
v.Decl = ArgumentDecl
p.scope.Declared = append(p.scope.Declared, v)
}
arrowFunc.Params.List = []BindingElement{{v, nil}}
arrowFunc.Body.List = p.parseArrowFuncBody()
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
return
}
func (p *Parser) parseArrowFuncBody() (list []IStmt) {
// expect we're at arrow
if p.tt != ArrowToken {
p.fail("arrow function", ArrowToken)
return
} else if p.prevLT {
p.fail("expression")
return
}
p.next()
// mark undeclared vars as arguments in `function f(a=b){var b}` where the b's are different vars
p.scope.MarkArguments()
if p.tt == OpenBraceToken {
parentInFor := p.inFor
p.inFor = false
p.allowDirectivePrologue = true
list = p.parseStmtList("arrow function")
p.inFor = parentInFor
} else {
list = []IStmt{&ReturnStmt{p.parseExpression(OpAssign)}}
}
return
}
func (p *Parser) parseIdentifierExpression(prec OpPrec, ident []byte) IExpr {
var left IExpr
left = p.scope.Use(ident)
return p.parseExpressionSuffix(left, prec, OpPrimary)
}
func (p *Parser) parseAsyncExpression(prec OpPrec, async []byte) IExpr {
// assume we're at a token after async
var left IExpr
precLeft := OpPrimary
if !p.prevLT && p.tt == FunctionToken {
// primary expression
left = p.parseAsyncFuncExpr()
} else if !p.prevLT && prec <= OpAssign && (p.tt == OpenParenToken || IsIdentifier(p.tt) || !p.generator && p.tt == YieldToken || p.tt == AwaitToken) {
// async arrow function expression
if p.tt == AwaitToken {
p.fail("arrow function")
return nil
} else if p.tt == OpenParenToken {
return p.parseParenthesizedExpressionOrArrowFunc(prec, async)
}
left = p.parseAsyncArrowFunc()
precLeft = OpAssign
} else {
left = p.scope.Use(async)
}
return p.parseExpressionSuffix(left, prec, precLeft)
}
// parseExpression parses an expression that has a precedence of prec or higher.
func (p *Parser) parseExpression(prec OpPrec) IExpr {
p.exprLevel++
if 1000 < p.exprLevel {
p.failMessage("too many nested expressions")
return nil
}
// reparse input if we have / or /= as the beginning of a new expression, this should be a regular expression!
if p.tt == DivToken || p.tt == DivEqToken {
p.tt, p.data = p.l.RegExp()
if p.tt == ErrorToken {
p.fail("regular expression")
return nil
}
}
var left IExpr
precLeft := OpPrimary
if IsIdentifier(p.tt) && p.tt != AsyncToken {
left = p.scope.Use(p.data)
p.next()
suffix := p.parseExpressionSuffix(left, prec, precLeft)
p.exprLevel--
return suffix
} else if IsNumeric(p.tt) {
left = &LiteralExpr{p.tt, p.data}
p.next()
suffix := p.parseExpressionSuffix(left, prec, precLeft)
p.exprLevel--
return suffix
}
switch tt := p.tt; tt {
case StringToken, ThisToken, NullToken, TrueToken, FalseToken, RegExpToken:
left = &LiteralExpr{p.tt, p.data}
p.next()
case OpenBracketToken:
parentInFor := p.inFor
p.inFor = false
array := p.parseArrayLiteral()
left = &array
p.inFor = parentInFor
case OpenBraceToken:
parentInFor := p.inFor
p.inFor = false
object := p.parseObjectLiteral()
left = &object
p.inFor = parentInFor
case OpenParenToken:
// parenthesized expression or arrow parameter list
if OpAssign < prec {
// must be a parenthesized expression
p.next()
parentInFor := p.inFor
p.inFor = false
left = &GroupExpr{p.parseExpression(OpExpr)}
p.inFor = parentInFor
if !p.consume("expression", CloseParenToken) {
return nil
}
break
}
suffix := p.parseParenthesizedExpressionOrArrowFunc(prec, nil)
p.exprLevel--
return suffix
case NotToken, BitNotToken, TypeofToken, VoidToken, DeleteToken:
if OpUnary < prec {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{tt, p.parseExpression(OpUnary)}
precLeft = OpUnary
case AddToken:
if OpUnary < prec {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{PosToken, p.parseExpression(OpUnary)}
precLeft = OpUnary
case SubToken:
if OpUnary < prec {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{NegToken, p.parseExpression(OpUnary)}
precLeft = OpUnary
case IncrToken:
if OpUpdate < prec {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{PreIncrToken, p.parseExpression(OpUnary)}
precLeft = OpUnary
case DecrToken:
if OpUpdate < prec {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{PreDecrToken, p.parseExpression(OpUnary)}
precLeft = OpUnary
case AwaitToken:
// either accepted as IdentifierReference or as AwaitExpression
if p.async && prec <= OpUnary {
p.next()
left = &UnaryExpr{tt, p.parseExpression(OpUnary)}
precLeft = OpUnary
} else if p.async {
p.fail("expression")
return nil
} else {
left = p.scope.Use(p.data)
p.next()
}
case NewToken:
p.next()
if p.tt == DotToken {
p.next()
if !p.consume("new.target expression", TargetToken) {
return nil
}
left = &NewTargetExpr{}
precLeft = OpMember
} else {
newExpr := &NewExpr{p.parseExpression(OpNew), nil}
if p.tt == OpenParenToken {
args := p.parseArguments()
if len(args.List) != 0 {
newExpr.Args = &args
}
precLeft = OpMember
} else {
precLeft = OpNew
}
left = newExpr
}
case ImportToken:
// OpMember < prec does never happen
left = &LiteralExpr{p.tt, p.data}
p.next()
if p.tt == DotToken {
p.next()
if !p.consume("import.meta expression", MetaToken) {
return nil
}
left = &ImportMetaExpr{}
precLeft = OpMember
} else if p.tt != OpenParenToken {
p.fail("import expression", OpenParenToken)
return nil
} else if OpCall < prec {
p.fail("expression")
return nil
} else {
precLeft = OpCall
}
case SuperToken:
// OpMember < prec does never happen
left = &LiteralExpr{p.tt, p.data}
p.next()
if OpCall < prec && p.tt != DotToken && p.tt != OpenBracketToken {
p.fail("super expression", OpenBracketToken, DotToken)
return nil
} else if p.tt != DotToken && p.tt != OpenBracketToken && p.tt != OpenParenToken {
p.fail("super expression", OpenBracketToken, OpenParenToken, DotToken)
return nil
}
if OpCall < prec {
precLeft = OpMember
} else {
precLeft = OpCall
}
case YieldToken:
// either accepted as IdentifierReference or as YieldExpression
if p.generator && prec <= OpAssign {
// YieldExpression
p.next()
yieldExpr := YieldExpr{}
if !p.prevLT {
yieldExpr.Generator = p.tt == MulToken
if yieldExpr.Generator {
p.next()
yieldExpr.X = p.parseExpression(OpAssign)
} else if p.tt != CloseBraceToken && p.tt != CloseBracketToken && p.tt != CloseParenToken && p.tt != ColonToken && p.tt != CommaToken && p.tt != SemicolonToken {
yieldExpr.X = p.parseExpression(OpAssign)
}
}
left = &yieldExpr
precLeft = OpAssign
} else if p.generator {
p.fail("expression")
return nil
} else {
left = p.scope.Use(p.data)
p.next()
}
case AsyncToken:
async := p.data
p.next()
left = p.parseAsyncExpression(prec, async)
case ClassToken:
parentInFor := p.inFor
p.inFor = false
left = p.parseClassExpr()
p.inFor = parentInFor
case FunctionToken:
parentInFor := p.inFor
p.inFor = false
left = p.parseFuncExpr()
p.inFor = parentInFor
case TemplateToken, TemplateStartToken:
parentInFor := p.inFor
p.inFor = false
template := p.parseTemplateLiteral(precLeft)
left = &template
p.inFor = parentInFor
default:
p.fail("expression")
return nil
}
suffix := p.parseExpressionSuffix(left, prec, precLeft)
p.exprLevel--
return suffix
}
func (p *Parser) parseExpressionSuffix(left IExpr, prec, precLeft OpPrec) IExpr {
for i := 0; ; i++ {
if 1000 < p.exprLevel+i {
p.failMessage("too many nested expressions")
return nil
}
switch tt := p.tt; tt {
case EqToken, MulEqToken, DivEqToken, ModEqToken, ExpEqToken, AddEqToken, SubEqToken, LtLtEqToken, GtGtEqToken, GtGtGtEqToken, BitAndEqToken, BitXorEqToken, BitOrEqToken, AndEqToken, OrEqToken, NullishEqToken:
if OpAssign < prec {
return left
} else if precLeft < OpLHS {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpAssign)}
precLeft = OpAssign
case LtToken, LtEqToken, GtToken, GtEqToken, InToken, InstanceofToken:
if OpCompare < prec || p.inFor && tt == InToken {
return left
} else if precLeft < OpCompare {
// can only fail after a yield or arrow function expression
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpShift)}
precLeft = OpCompare
case EqEqToken, NotEqToken, EqEqEqToken, NotEqEqToken:
if OpEquals < prec {
return left
} else if precLeft < OpEquals {
// can only fail after a yield or arrow function expression
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpCompare)}
precLeft = OpEquals
case AndToken:
if OpAnd < prec {
return left
} else if precLeft < OpAnd {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpBitOr)}
precLeft = OpAnd
case OrToken:
if OpOr < prec {
return left
} else if precLeft < OpOr {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpAnd)}
precLeft = OpOr
case NullishToken:
if OpCoalesce < prec {
return left
} else if precLeft < OpBitOr && precLeft != OpCoalesce {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpBitOr)}
precLeft = OpCoalesce
case DotToken:
// OpMember < prec does never happen
if precLeft < OpCall {
p.fail("expression")
return nil
}
p.next()
if !IsIdentifierName(p.tt) && p.tt != PrivateIdentifierToken {
p.fail("dot expression", IdentifierToken)
return nil
}
exprPrec := OpMember
if precLeft < OpMember {
exprPrec = OpCall
}
if p.tt != PrivateIdentifierToken {
p.tt = IdentifierToken
}
left = &DotExpr{left, LiteralExpr{p.tt, p.data}, exprPrec}
p.next()
if precLeft < OpMember {
precLeft = OpCall
} else {
precLeft = OpMember
}
case OpenBracketToken:
// OpMember < prec does never happen
if precLeft < OpCall {
p.fail("expression")
return nil
}
p.next()
exprPrec := OpMember
if precLeft < OpMember {
exprPrec = OpCall
}
parentInFor := p.inFor
p.inFor = false
left = &IndexExpr{left, p.parseExpression(OpExpr), exprPrec}
p.inFor = parentInFor
if !p.consume("index expression", CloseBracketToken) {
return nil
}
if precLeft < OpMember {
precLeft = OpCall
} else {
precLeft = OpMember
}
case OpenParenToken:
if OpCall < prec {
return left
} else if precLeft < OpCall {
p.fail("expression")
return nil
}
parentInFor := p.inFor
p.inFor = false
left = &CallExpr{left, p.parseArguments()}
precLeft = OpCall
p.inFor = parentInFor
case TemplateToken, TemplateStartToken:
// OpMember < prec does never happen
if precLeft < OpCall {
p.fail("expression")
return nil
}
parentInFor := p.inFor
p.inFor = false
template := p.parseTemplateLiteral(precLeft)
template.Tag = left
left = &template
if precLeft < OpMember {
precLeft = OpCall
} else {
precLeft = OpMember
}
p.inFor = parentInFor
case OptChainToken:
if OpCall < prec {
return left
}
p.next()
if p.tt == OpenParenToken {
left = &OptChainExpr{left, &CallExpr{nil, p.parseArguments()}}
} else if p.tt == OpenBracketToken {
p.next()
left = &OptChainExpr{left, &IndexExpr{nil, p.parseExpression(OpExpr), OpCall}}
if !p.consume("optional chaining expression", CloseBracketToken) {
return nil
}
} else if p.tt == TemplateToken || p.tt == TemplateStartToken {
template := p.parseTemplateLiteral(precLeft)
left = &OptChainExpr{left, &template}
} else if IsIdentifierName(p.tt) {
left = &OptChainExpr{left, &LiteralExpr{IdentifierToken, p.data}}
p.next()
} else if p.tt == PrivateIdentifierToken {
left = &OptChainExpr{left, &LiteralExpr{p.tt, p.data}}
p.next()
} else {
p.fail("optional chaining expression", IdentifierToken, OpenParenToken, OpenBracketToken, TemplateToken)
return nil
}
precLeft = OpCall
case IncrToken:
if p.prevLT || OpUpdate < prec {
return left
} else if precLeft < OpLHS {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{PostIncrToken, left}
precLeft = OpUpdate
case DecrToken:
if p.prevLT || OpUpdate < prec {
return left
} else if precLeft < OpLHS {
p.fail("expression")
return nil
}
p.next()
left = &UnaryExpr{PostDecrToken, left}
precLeft = OpUpdate
case ExpToken:
if OpExp < prec {
return left
} else if precLeft < OpUpdate {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpExp)}
precLeft = OpExp
case MulToken, DivToken, ModToken:
if OpMul < prec {
return left
} else if precLeft < OpMul {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpExp)}
precLeft = OpMul
case AddToken, SubToken:
if OpAdd < prec {
return left
} else if precLeft < OpAdd {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpMul)}
precLeft = OpAdd
case LtLtToken, GtGtToken, GtGtGtToken:
if OpShift < prec {
return left
} else if precLeft < OpShift {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpAdd)}
precLeft = OpShift
case BitAndToken:
if OpBitAnd < prec {
return left
} else if precLeft < OpBitAnd {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpEquals)}
precLeft = OpBitAnd
case BitXorToken:
if OpBitXor < prec {
return left
} else if precLeft < OpBitXor {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpBitAnd)}
precLeft = OpBitXor
case BitOrToken:
if OpBitOr < prec {
return left
} else if precLeft < OpBitOr {
p.fail("expression")
return nil
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpBitXor)}
precLeft = OpBitOr
case QuestionToken:
if OpAssign < prec {
return left
} else if precLeft < OpCoalesce {
p.fail("expression")
return nil
}
p.next()
ifExpr := p.parseExpression(OpAssign)
if !p.consume("conditional expression", ColonToken) {
return nil
}
elseExpr := p.parseExpression(OpAssign)
left = &CondExpr{left, ifExpr, elseExpr}
precLeft = OpAssign
case CommaToken:
if OpExpr < prec {
return left
}
p.next()
left = &BinaryExpr{tt, left, p.parseExpression(OpAssign)}
precLeft = OpExpr
case ArrowToken:
// handle identifier => ..., where identifier could also be yield or await
if OpAssign < prec {
return left
} else if precLeft < OpPrimary {
p.fail("expression")
return nil
}
v, ok := left.(*Var)
if !ok {
p.fail("expression")
return nil
}
left = p.parseIdentifierArrowFunc(v)
precLeft = OpAssign
default:
return left
}
}
}
func (p *Parser) parseAssignmentExpression() IExpr {
// this could be a BindingElement or an AssignmentExpression. Here we handle BindingIdentifier with a possible Initializer, BindingPattern will be handled by parseArrayLiteral or parseObjectLiteral
if p.assumeArrowFunc && p.isIdentifierReference(p.tt) {
tt := p.tt
data := p.data
p.next()
if p.tt == EqToken || p.tt == CommaToken || p.tt == CloseParenToken || p.tt == CloseBraceToken || p.tt == CloseBracketToken {
var left IExpr
left, _ = p.scope.Declare(ArgumentDecl, data) // cannot fail
p.assumeArrowFunc = false
left = p.parseExpressionSuffix(left, OpAssign, OpPrimary)
p.assumeArrowFunc = true
return left
}
p.assumeArrowFunc = false
if tt == AsyncToken {
return p.parseAsyncExpression(OpAssign, data)
}
return p.parseIdentifierExpression(OpAssign, data)
} else if p.tt != OpenBracketToken && p.tt != OpenBraceToken {
p.assumeArrowFunc = false
}
return p.parseExpression(OpAssign)
}
func (p *Parser) parseParenthesizedExpressionOrArrowFunc(prec OpPrec, async []byte) IExpr {
var left IExpr
precLeft := OpPrimary
// expect to be at (
p.next()
isAsync := async != nil
arrowFunc := &ArrowFunc{}
parent := p.enterScope(&arrowFunc.Body.Scope, true)
parentAssumeArrowFunc, parentInFor := p.assumeArrowFunc, p.inFor
p.assumeArrowFunc, p.inFor = true, false
// parse a parenthesized expression but assume we might be parsing an (async) arrow function. If this is really an arrow function, parsing as a parenthesized expression cannot fail as AssignmentExpression, ArrayLiteral, and ObjectLiteral are supersets of SingleNameBinding, ArrayBindingPattern, and ObjectBindingPattern respectively. Any identifier that would be a BindingIdentifier in case of an arrow function, will be added as such. If finally this is not an arrow function, we will demote those variables an undeclared and merge them with the parent scope.
var list []IExpr
var rest IExpr
for p.tt != CloseParenToken && p.tt != ErrorToken {
if p.tt == EllipsisToken && p.assumeArrowFunc {
p.next()
if isAsync {
rest = p.parseAssignmentExpression()
if p.tt == CommaToken {
p.next()
}
} else if p.isIdentifierReference(p.tt) {
rest, _ = p.scope.Declare(ArgumentDecl, p.data) // cannot fail
p.next()
} else if p.tt == OpenBracketToken {
array := p.parseArrayLiteral()
rest = &array
} else if p.tt == OpenBraceToken {
object := p.parseObjectLiteral()
rest = &object
} else {
p.fail("arrow function")
return nil
}
break
}
list = append(list, p.parseAssignmentExpression())
if p.tt != CommaToken {
break
}
p.next()
}
if p.tt != CloseParenToken {
p.fail("expression")
return nil
}
p.next()
isArrowFunc := p.tt == ArrowToken && p.assumeArrowFunc
p.assumeArrowFunc, p.inFor = parentAssumeArrowFunc, parentInFor
if isArrowFunc {
parentAsync, parentGenerator := p.async, p.generator
p.async, p.generator = isAsync, false
// arrow function
arrowFunc.Params = Params{List: make([]BindingElement, len(list))}
for i, item := range list {
arrowFunc.Params.List[i] = p.exprToBindingElement(item) // can not fail when assumArrowFunc is set
}
arrowFunc.Async = isAsync
arrowFunc.Params.Rest = p.exprToBinding(rest)
arrowFunc.Body.List = p.parseArrowFuncBody()
p.async, p.generator = parentAsync, parentGenerator
p.exitScope(parent)
left = arrowFunc
precLeft = OpAssign
} else if len(list) == 0 || !isAsync && rest != nil || isAsync && OpCall < prec {
p.fail("arrow function", ArrowToken)
return nil
} else {
p.exitScope(parent)
// for any nested FuncExpr/ArrowFunc scope, Parent will point to the temporary scope created in case this was an arrow function instead of a parenthesized expression. This is not a problem as Parent is only used for defining new variables, and we already parsed all the nested scopes so that Parent (not Func) are not relevant anymore. Anyways, the Parent will just point to an empty scope, whose Parent/Func will point to valid scopes. This should not be a big deal.
// Here we move all declared ArgumentDecls (in case of an arrow function) to its parent scope as undeclared variables (identifiers used in a parenthesized expression).
arrowFunc.Body.Scope.UndeclareScope()
if isAsync {
// call expression
args := Args{}
for _, item := range list {
args.List = append(args.List, Arg{Value: item, Rest: false})
}
if rest != nil {
args.List = append(args.List, Arg{Value: rest, Rest: true})
}
left = p.scope.Use(async)
left = &CallExpr{left, args}
precLeft = OpCall
} else {
// parenthesized expression
left = list[0]
for _, item := range list[1:] {
left = &BinaryExpr{CommaToken, left, item}
}
left = &GroupExpr{left}
}
}
return p.parseExpressionSuffix(left, prec, precLeft)
}
// exprToBinding converts a CoverParenthesizedExpressionAndArrowParameterList into FormalParameters
// Any unbound variables of the parameters (Initializer, ComputedPropertyName) are kept in the parent scope
func (p *Parser) exprToBinding(expr IExpr) (binding IBinding) {
if v, ok := expr.(*Var); ok {
binding = v
} else if array, ok := expr.(*ArrayExpr); ok {
bindingArray := BindingArray{}
for _, item := range array.List {
if item.Spread {
// can only BindingIdentifier or BindingPattern
bindingArray.Rest = p.exprToBinding(item.Value)
break
}
var bindingElement BindingElement
bindingElement = p.exprToBindingElement(item.Value)
bindingArray.List = append(bindingArray.List, bindingElement)
}
binding = &bindingArray
} else if object, ok := expr.(*ObjectExpr); ok {
bindingObject := BindingObject{}
for _, item := range object.List {
if item.Spread {
// can only be BindingIdentifier
bindingObject.Rest = item.Value.(*Var)
break
}
var bindingElement BindingElement
bindingElement.Binding = p.exprToBinding(item.Value)
if bindingElement.Binding == nil {
bindingElement = p.exprToBindingElement(item.Value)
} else if item.Init != nil {
bindingElement.Default = item.Init
}
bindingObject.List = append(bindingObject.List, BindingObjectItem{Key: item.Name, Value: bindingElement})
}
binding = &bindingObject
}
return
}
func (p *Parser) exprToBindingElement(expr IExpr) (bindingElement BindingElement) {
if assign, ok := expr.(*BinaryExpr); ok && assign.Op == EqToken {
bindingElement.Binding = p.exprToBinding(assign.X)
bindingElement.Default = assign.Y
} else {
bindingElement.Binding = p.exprToBinding(expr)
}
return
}
func (p *Parser) isIdentifierReference(tt TokenType) bool {
return IsIdentifier(tt) || tt == YieldToken && !p.generator || tt == AwaitToken && !p.async
}
parse-2.5.21/js/parse_test.go 0000664 0000000 0000000 00000145307 14117006534 0016042 0 ustar 00root root 0000000 0000000 package js
import (
"fmt"
"io"
"sort"
"strings"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
////////////////////////////////////////////////////////////////
func TestParse(t *testing.T) {
var tests = []struct {
js string
expected string
}{
// grammar
{"", ""},
{"\n", ""},
{"/* comment */", ""},
{"{}", "Stmt({ })"},
{`"use strict"`, `Stmt("use strict")`},
{"var a = b;", "Decl(var Binding(a = b))"},
{"const a = b;", "Decl(const Binding(a = b))"},
{"let a = b;", "Decl(let Binding(a = b))"},
{"let [a,b] = [1, 2];", "Decl(let Binding([ Binding(a), Binding(b) ] = [1, 2]))"},
{"let [a,[b,c]] = [1, [2, 3]];", "Decl(let Binding([ Binding(a), Binding([ Binding(b), Binding(c) ]) ] = [1, [2, 3]]))"},
{"let [,,c] = [1, 2, 3];", "Decl(let Binding([ Binding(), Binding(), Binding(c) ] = [1, 2, 3]))"},
{"let [a, ...b] = [1, 2, 3];", "Decl(let Binding([ Binding(a), ...Binding(b) ] = [1, 2, 3]))"},
{"let {a, b} = {a: 3, b: 4};", "Decl(let Binding({ Binding(a), Binding(b) } = {a: 3, b: 4}))"},
{"let {a: [b, {c}]} = {a: [5, {c: 3}]};", "Decl(let Binding({ a: Binding([ Binding(b), Binding({ Binding(c) }) ]) } = {a: [5, {c: 3}]}))"},
{"let [a = 2] = [];", "Decl(let Binding([ Binding(a = 2) ] = []))"},
{"let {a: b = 2} = {};", "Decl(let Binding({ a: Binding(b = 2) } = {}))"},
{"var a = 5 * 4 / 3 ** 2 + ( 5 - 3 );", "Decl(var Binding(a = (((5*4)/(3**2))+((5-3)))))"},
{"var a, b = c;", "Decl(var Binding(a) Binding(b = c))"},
{"var a,\nb = c;", "Decl(var Binding(a) Binding(b = c))"},
{";", "Stmt(;)"},
{"{; var a = 3;}", "Stmt({ Stmt(;) Decl(var Binding(a = 3)) })"},
{"{a=5}", "Stmt({ Stmt(a=5) })"},
{"return", "Stmt(return)"},
{"return 5*3", "Stmt(return (5*3))"},
{"break", "Stmt(break)"},
{"break LABEL", "Stmt(break LABEL)"},
{"continue", "Stmt(continue)"},
{"continue LABEL", "Stmt(continue LABEL)"},
{"if (a == 5) return true", "Stmt(if (a==5) Stmt(return true))"},
{"if (a == 5) return true else return false", "Stmt(if (a==5) Stmt(return true) else Stmt(return false))"},
{"if (a) b; else if (c) d;", "Stmt(if a Stmt(b) else Stmt(if c Stmt(d)))"},
{"if (a) 1; else if (b) 2; else 3", "Stmt(if a Stmt(1) else Stmt(if b Stmt(2) else Stmt(3)))"},
{"with (a = 5) return true", "Stmt(with (a=5) Stmt(return true))"},
{"do a++; while (a < 4)", "Stmt(do Stmt(a++) while (a<4))"},
{"do {a++} while (a < 4)", "Stmt(do Stmt({ Stmt(a++) }) while (a<4))"},
{"while (a < 4) a++", "Stmt(while (a<4) Stmt(a++))"},
{"for (var a = 0; a < 4; a++) b = a", "Stmt(for Decl(var Binding(a = 0)) ; (a<4) ; (a++) Stmt({ Stmt(b=a) }))"},
{"for (5; a < 4; a++) {}", "Stmt(for 5 ; (a<4) ; (a++) Stmt({ }))"},
{"for (;;) {}", "Stmt(for ; ; Stmt({ }))"},
{"for (a,b=5;;) {}", "Stmt(for (a,(b=5)) ; ; Stmt({ }))"},
{"for (let a;;) {}", "Stmt(for Decl(let Binding(a)) ; ; Stmt({ }))"},
{"for (var a in b) {}", "Stmt(for Decl(var Binding(a)) in b Stmt({ }))"},
{"for (var a in b) c", "Stmt(for Decl(var Binding(a)) in b Stmt({ Stmt(c) }))"},
{"for (var a of b) {}", "Stmt(for Decl(var Binding(a)) of b Stmt({ }))"},
{"for (var a of b) c", "Stmt(for Decl(var Binding(a)) of b Stmt({ Stmt(c) }))"},
{"for ((b in c) in d) {}", "Stmt(for ((b in c)) in d Stmt({ }))"},
{"for (c && (a in b);;) {}", "Stmt(for (c&&((a in b))) ; ; Stmt({ }))"},
{"for (a in b) {}", "Stmt(for a in b Stmt({ }))"},
{"for (a = b;;) {}", "Stmt(for (a=b) ; ; Stmt({ }))"},
{"for (var [a] in b) {}", "Stmt(for Decl(var Binding([ Binding(a) ])) in b Stmt({ }))"},
{"throw 5", "Stmt(throw 5)"},
{"try {} catch {b}", "Stmt(try Stmt({ }) catch Stmt({ Stmt(b) }))"},
{"try {} finally {c}", "Stmt(try Stmt({ }) finally Stmt({ Stmt(c) }))"},
{"try {} catch {b} finally {c}", "Stmt(try Stmt({ }) catch Stmt({ Stmt(b) }) finally Stmt({ Stmt(c) }))"},
{"try {} catch (e) {b}", "Stmt(try Stmt({ }) catch Binding(e) Stmt({ Stmt(b) }))"},
{"debugger", "Stmt(debugger)"},
{"label: var a", "Stmt(label : Decl(var Binding(a)))"},
{"yield: var a", "Stmt(yield : Decl(var Binding(a)))"},
{"await: var a", "Stmt(await : Decl(var Binding(a)))"},
{"switch (5) {}", "Stmt(switch 5)"},
{"switch (5) { case 3: {} default: {}}", "Stmt(switch 5 Clause(case 3 Stmt({ })) Clause(default Stmt({ })))"},
{"function a(b) {}", "Decl(function a Params(Binding(b)) Stmt({ }))"},
{"async function a(b) {}", "Decl(async function a Params(Binding(b)) Stmt({ }))"},
{"function* a(b) {}", "Decl(function* a Params(Binding(b)) Stmt({ }))"},
{"function a(b,) {}", "Decl(function a Params(Binding(b)) Stmt({ }))"},
{"function a(b, c) {}", "Decl(function a Params(Binding(b), Binding(c)) Stmt({ }))"},
{"function a(...b) {}", "Decl(function a Params(...Binding(b)) Stmt({ }))"},
{"function a(b, ...c) {}", "Decl(function a Params(Binding(b), ...Binding(c)) Stmt({ }))"},
{"function a(b) {return}", "Decl(function a Params(Binding(b)) Stmt({ Stmt(return) }))"},
{"class A {}", "Decl(class A)"},
{"class A {;}", "Decl(class A)"},
{"!class{}", "Stmt(!Decl(class))"},
{"class A extends B { }", "Decl(class A extends B)"},
{"class A { a(b) {} }", "Decl(class A Method(a Params(Binding(b)) Stmt({ })))"},
{"class A { 5(b) {} }", "Decl(class A Method(5 Params(Binding(b)) Stmt({ })))"},
{"class A { 'a'(b) {} }", "Decl(class A Method(a Params(Binding(b)) Stmt({ })))"},
{"class A { '5'(b) {} }", "Decl(class A Method(5 Params(Binding(b)) Stmt({ })))"},
{"class A { '%'(b) {} }", "Decl(class A Method('%' Params(Binding(b)) Stmt({ })))"},
{"class A { get() {} }", "Decl(class A Method(get Params() Stmt({ })))"},
{"class A { get a() {} }", "Decl(class A Method(get a Params() Stmt({ })))"},
{"class A { set a(b) {} }", "Decl(class A Method(set a Params(Binding(b)) Stmt({ })))"},
{"class A { * a(b) {} }", "Decl(class A Method(* a Params(Binding(b)) Stmt({ })))"},
{"class A { async a(b) {} }", "Decl(class A Method(async a Params(Binding(b)) Stmt({ })))"},
{"class A { async * a(b) {} }", "Decl(class A Method(async * a Params(Binding(b)) Stmt({ })))"},
{"class A { static() {} }", "Decl(class A Method(static Params() Stmt({ })))"},
{"class A { static a(b) {} }", "Decl(class A Method(static a Params(Binding(b)) Stmt({ })))"},
{"class A { [5](b) {} }", "Decl(class A Method([5] Params(Binding(b)) Stmt({ })))"},
{"class A { field }", "Decl(class A Definition(field))"},
{"class A { #field }", "Decl(class A Definition(#field))"},
{"class A { field=5 }", "Decl(class A Definition(field = 5))"},
{"class A { #field=5 }", "Decl(class A Definition(#field = 5))"},
{"class A { get }", "Decl(class A Definition(get))"},
{"class A { field static get method(){} }", "Decl(class A Definition(field) Method(static get method Params() Stmt({ })))"},
//{"class A { get get get(){} }", "Decl(class A Definition(get) Method(get get Params() Stmt({ })))"}, // doesn't look like this should be supported
{"`tmpl`", "Stmt(`tmpl`)"},
{"`tmpl${x}`", "Stmt(`tmpl${x}`)"},
{"`tmpl${x}tmpl${x}`", "Stmt(`tmpl${x}tmpl${x}`)"},
{"import \"pkg\";", "Stmt(import \"pkg\")"},
{"import yield from \"pkg\"", "Stmt(import yield from \"pkg\")"},
{"import * as yield from \"pkg\"", "Stmt(import * as yield from \"pkg\")"},
{"import {yield, for as yield,} from \"pkg\"", "Stmt(import { yield , for as yield , } from \"pkg\")"},
{"import yield, * as yield from \"pkg\"", "Stmt(import yield , * as yield from \"pkg\")"},
{"import yield, {yield} from \"pkg\"", "Stmt(import yield , { yield } from \"pkg\")"},
{"import {yield,} from \"pkg\"", "Stmt(import { yield , } from \"pkg\")"},
{"export * from \"pkg\";", "Stmt(export * from \"pkg\")"},
{"export * as for from \"pkg\"", "Stmt(export * as for from \"pkg\")"},
{"export {if, for as switch} from \"pkg\"", "Stmt(export { if , for as switch } from \"pkg\")"},
{"export {if, for as switch,}", "Stmt(export { if , for as switch , })"},
{"export var a", "Stmt(export Decl(var Binding(a)))"},
{"export function a(b){}", "Stmt(export Decl(function a Params(Binding(b)) Stmt({ })))"},
{"export async function a(b){}", "Stmt(export Decl(async function a Params(Binding(b)) Stmt({ })))"},
{"export class A{}", "Stmt(export Decl(class A))"},
{"export default function(b){}", "Stmt(export default Decl(function Params(Binding(b)) Stmt({ })))"},
{"export default async function(b){}", "Stmt(export default Decl(async function Params(Binding(b)) Stmt({ })))"},
{"export default class{}", "Stmt(export default Decl(class))"},
{"export default a", "Stmt(export default a)"},
{"export default async", "Stmt(export default async)"},
// yield, await, async
{"yield\na = 5", "Stmt(yield) Stmt(a=5)"},
{"yield * yield * a", "Stmt((yield*yield)*a)"},
{"(yield) => 5", "Stmt(Params(Binding(yield)) => Stmt({ Stmt(return 5) }))"},
{"(await) => 5", "Stmt(Params(Binding(await)) => Stmt({ Stmt(return 5) }))"},
{"async", "Stmt(async)"},
{"async = a", "Stmt(async=a)"},
{"async\n= a", "Stmt(async=a)"},
{"async a => b", "Stmt(async Params(Binding(a)) => Stmt({ Stmt(return b) }))"},
{"async (a) => b", "Stmt(async Params(Binding(a)) => Stmt({ Stmt(return b) }))"},
{"async(a)", "Stmt(async(a))"},
{"async(a=6, ...b)", "Stmt(async((a=6), ...b))"},
{"async(function(){})", "Stmt(async(Decl(function Params() Stmt({ }))))"},
{"async\nawait => b", "Stmt(async) Stmt(Params(Binding(await)) => Stmt({ Stmt(return b) }))"},
{"a + async\nb", "Stmt(a+async) Stmt(b)"},
{"a + async\nfunction f(){}", "Stmt(a+async) Decl(function f Params() Stmt({ }))"},
{"class a extends async {}", "Decl(class a extends async)"},
{"function*a(){ yield a = 5 }", "Decl(function* a Params() Stmt({ Stmt(yield (a=5)) }))"},
{"function*a(){ yield * a = 5 }", "Decl(function* a Params() Stmt({ Stmt(yield* (a=5)) }))"},
{"function*a(){ yield\na = 5 }", "Decl(function* a Params() Stmt({ Stmt(yield) Stmt(a=5) }))"},
{"function*a(){ yield yield a }", "Decl(function* a Params() Stmt({ Stmt(yield (yield a)) }))"},
{"function*a(){ yield * yield * a }", "Decl(function* a Params() Stmt({ Stmt(yield* (yield* a)) }))"},
{"function*a(b = yield c){}", "Decl(function* a Params(Binding(b = (yield c))) Stmt({ }))"},
{"function*a(){ x = function yield(){} }", "Decl(function* a Params() Stmt({ Stmt(x=Decl(function yield Params() Stmt({ }))) }))"},
{"function*a(){ x = function b(){ x = yield } }", "Decl(function* a Params() Stmt({ Stmt(x=Decl(function b Params() Stmt({ Stmt(x=yield) }))) }))"},
{"function*a(){ (yield) }", "Decl(function* a Params() Stmt({ Stmt((yield)) }))"},
{"function*a(){ (yield a) }", "Decl(function* a Params() Stmt({ Stmt((yield a)) }))"},
{"let\nawait", "Decl(let Binding(await))"},
{"x = {await}", "Stmt(x={await})"},
{"async function a(){ x = {await: 5} }", "Decl(async function a Params() Stmt({ Stmt(x={await: 5}) }))"},
{"async function a(){ x = await a }", "Decl(async function a Params() Stmt({ Stmt(x=(await a)) }))"},
{"async function a(){ x = await a+y }", "Decl(async function a Params() Stmt({ Stmt(x=((await a)+y)) }))"},
{"async function a(b = await c){}", "Decl(async function a Params(Binding(b = (await c))) Stmt({ }))"},
{"async function a(){ x = function await(){} }", "Decl(async function a Params() Stmt({ Stmt(x=Decl(function await Params() Stmt({ }))) }))"},
{"async function a(){ x = function b(){ x = await } }", "Decl(async function a Params() Stmt({ Stmt(x=Decl(function b Params() Stmt({ Stmt(x=await) }))) }))"},
{"async function a(){ for await (var a of b) {} }", "Decl(async function a Params() Stmt({ Stmt(for await Decl(var Binding(a)) of b Stmt({ })) }))"},
{"async function a(){ (await a) }", "Decl(async function a Params() Stmt({ Stmt((await a)) }))"},
{"x = {async a(b){}}", "Stmt(x={Method(async a Params(Binding(b)) Stmt({ }))})"},
// bindings
{"let [] = z", "Decl(let Binding([ ] = z))"},
{"let [,] = z", "Decl(let Binding([ ] = z))"},
{"let [,a] = z", "Decl(let Binding([ Binding(), Binding(a) ] = z))"},
{"let [name = 5] = z", "Decl(let Binding([ Binding(name = 5) ] = z))"},
{"let [name = 5,] = z", "Decl(let Binding([ Binding(name = 5) ] = z))"},
{"let [name = 5,,] = z", "Decl(let Binding([ Binding(name = 5) ] = z))"},
{"let [name = 5,, ...yield] = z", "Decl(let Binding([ Binding(name = 5), Binding(), ...Binding(yield) ] = z))"},
{"let [...yield] = z", "Decl(let Binding([ ...Binding(yield) ] = z))"},
{"let [,,...yield] = z", "Decl(let Binding([ Binding(), Binding(), ...Binding(yield) ] = z))"},
{"let [name = 5,, ...[yield]] = z", "Decl(let Binding([ Binding(name = 5), Binding(), ...Binding([ Binding(yield) ]) ] = z))"},
{"let [name = 5,, ...{yield}] = z", "Decl(let Binding([ Binding(name = 5), Binding(), ...Binding({ Binding(yield) }) ] = z))"},
{"let {} = z", "Decl(let Binding({ } = z))"},
{"let {name = 5} = z", "Decl(let Binding({ Binding(name = 5) } = z))"},
{"let {await = 5} = z", "Decl(let Binding({ Binding(await = 5) } = z))"},
{"let {if: name} = z", "Decl(let Binding({ if: Binding(name) } = z))"},
{"let {\"string\": name} = z", "Decl(let Binding({ string: Binding(name) } = z))"},
{"let {[a = 5]: name} = z", "Decl(let Binding({ [a=5]: Binding(name) } = z))"},
{"let {if: name = 5} = z", "Decl(let Binding({ if: Binding(name = 5) } = z))"},
{"let {if: yield = 5} = z", "Decl(let Binding({ if: Binding(yield = 5) } = z))"},
{"let {if: [name] = 5} = z", "Decl(let Binding({ if: Binding([ Binding(name) ] = 5) } = z))"},
{"let {if: {name} = 5} = z", "Decl(let Binding({ if: Binding({ Binding(name) } = 5) } = z))"},
{"let {...yield} = z", "Decl(let Binding({ ...Binding(yield) } = z))"},
{"let {if: name, ...yield} = z", "Decl(let Binding({ if: Binding(name), ...Binding(yield) } = z))"},
{"let i;for(let i;;);", "Decl(let Binding(i)) Stmt(for Decl(let Binding(i)) ; ; Stmt({ }))"},
{"let i;for(let i in x);", "Decl(let Binding(i)) Stmt(for Decl(let Binding(i)) in x Stmt({ }))"},
{"let i;for(let i of x);", "Decl(let Binding(i)) Stmt(for Decl(let Binding(i)) of x Stmt({ }))"},
{"for(let a in [0,1,2]){let a=5}", "Stmt(for Decl(let Binding(a)) in [0, 1, 2] Stmt({ Decl(let Binding(a = 5)) }))"},
{"for(var a in [0,1,2]){let a=5}", "Stmt(for Decl(var Binding(a)) in [0, 1, 2] Stmt({ Decl(let Binding(a = 5)) }))"},
{"for(var a in [0,1,2]){var a=5}", "Stmt(for Decl(var Binding(a)) in [0, 1, 2] Stmt({ Decl(var Binding(a = 5)) }))"},
{"for(let a=0; a<10; a++){let a=5}", "Stmt(for Decl(let Binding(a = 0)) ; (a<10) ; (a++) Stmt({ Decl(let Binding(a = 5)) }))"},
{"for(var a=0; a<10; a++){let a=5}", "Stmt(for Decl(var Binding(a = 0)) ; (a<10) ; (a++) Stmt({ Decl(let Binding(a = 5)) }))"},
{"for(var a=0; a<10; a++){var a=5}", "Stmt(for Decl(var Binding(a = 0)) ; (a<10) ; (a++) Stmt({ Decl(var Binding(a = 5)) }))"},
// expressions
{"x = [a, ...b]", "Stmt(x=[a, ...b])"},
{"x = [...b]", "Stmt(x=[...b])"},
{"x = [...a, ...b]", "Stmt(x=[...a, ...b])"},
{"x = [,]", "Stmt(x=[,])"},
{"x = [,,]", "Stmt(x=[, ,])"},
{"x = [a,]", "Stmt(x=[a])"},
{"x = [a,,]", "Stmt(x=[a, ,])"},
{"x = [,a]", "Stmt(x=[, a])"},
{"x = {a}", "Stmt(x={a})"},
{"x = {...a}", "Stmt(x={...a})"},
{"x = {a, ...b}", "Stmt(x={a, ...b})"},
{"x = {...a, ...b}", "Stmt(x={...a, ...b})"},
{"x = {a=5}", "Stmt(x={a = 5})"},
{"x = {yield=5}", "Stmt(x={yield = 5})"},
{"x = {a:5}", "Stmt(x={a: 5})"},
{"x = {yield:5}", "Stmt(x={yield: 5})"},
{"x = {async:5}", "Stmt(x={async: 5})"},
{"x = {if:5}", "Stmt(x={if: 5})"},
{"x = {\"string\":5}", "Stmt(x={string: 5})"},
{"x = {3:5}", "Stmt(x={3: 5})"},
{"x = {[3]:5}", "Stmt(x={[3]: 5})"},
{"x = {a, if: b, do(){}, ...d}", "Stmt(x={a, if: b, Method(do Params() Stmt({ })), ...d})"},
{"x = {*a(){}}", "Stmt(x={Method(* a Params() Stmt({ }))})"},
{"x = {async*a(){}}", "Stmt(x={Method(async * a Params() Stmt({ }))})"},
{"x = {get a(){}}", "Stmt(x={Method(get a Params() Stmt({ }))})"},
{"x = {set a(){}}", "Stmt(x={Method(set a Params() Stmt({ }))})"},
{"x = {get(){}}", "Stmt(x={Method(get Params() Stmt({ }))})"},
{"x = {set(){}}", "Stmt(x={Method(set Params() Stmt({ }))})"},
{"x = (a, b)", "Stmt(x=((a,b)))"},
{"x = function() {}", "Stmt(x=Decl(function Params() Stmt({ })))"},
{"x = async function() {}", "Stmt(x=Decl(async function Params() Stmt({ })))"},
{"x = class {}", "Stmt(x=Decl(class))"},
{"x = class {a(){}}", "Stmt(x=Decl(class Method(a Params() Stmt({ }))))"},
{"x = a => a++", "Stmt(x=(Params(Binding(a)) => Stmt({ Stmt(return (a++)) })))"},
{"x = a => {a++}", "Stmt(x=(Params(Binding(a)) => Stmt({ Stmt(a++) })))"},
{"x = a => {return}", "Stmt(x=(Params(Binding(a)) => Stmt({ Stmt(return) })))"},
{"x = a => {return a}", "Stmt(x=(Params(Binding(a)) => Stmt({ Stmt(return a) })))"},
{"x = yield => a++", "Stmt(x=(Params(Binding(yield)) => Stmt({ Stmt(return (a++)) })))"},
{"x = yield => {a++}", "Stmt(x=(Params(Binding(yield)) => Stmt({ Stmt(a++) })))"},
{"x = async a => a++", "Stmt(x=(async Params(Binding(a)) => Stmt({ Stmt(return (a++)) })))"},
{"x = async a => {a++}", "Stmt(x=(async Params(Binding(a)) => Stmt({ Stmt(a++) })))"},
{"x = async a => await b", "Stmt(x=(async Params(Binding(a)) => Stmt({ Stmt(return (await b)) })))"},
{"x = await => a++", "Stmt(x=(Params(Binding(await)) => Stmt({ Stmt(return (a++)) })))"},
{"x = a??b", "Stmt(x=(a??b))"},
{"x = a[b]", "Stmt(x=(a[b]))"},
{"x = a?.b?.c.d", "Stmt(x=(((a?.b)?.c).d))"},
{"x = a?.[b]?.`tpl`", "Stmt(x=((a?.[b])?.`tpl`))"},
{"x = a?.(b)", "Stmt(x=(a?.(b)))"},
{"x = super(a)", "Stmt(x=(super(a)))"},
{"x = a(a,b,...c,)", "Stmt(x=(a(a, b, ...c)))"},
{"x = a(...a,...b)", "Stmt(x=(a(...a, ...b)))"},
{"x = new a", "Stmt(x=(new a))"},
{"x = new a()", "Stmt(x=(new a))"},
{"x = new a(b)", "Stmt(x=(new a(b)))"},
{"x = new a().b(c)", "Stmt(x=(((new a).b)(c)))"},
{"x = new new.target", "Stmt(x=(new (new.target)))"},
{"x = new import.meta", "Stmt(x=(new (import.meta)))"},
{"x = import(a)", "Stmt(x=(import(a)))"},
{"import('module')", "Stmt(import('module'))"},
{"x = +a", "Stmt(x=(+a))"},
{"x = ++a", "Stmt(x=(++a))"},
{"x = -a", "Stmt(x=(-a))"},
{"x = --a", "Stmt(x=(--a))"},
{"x = a--", "Stmt(x=(a--))"},
{"x = a<<b", "Stmt(x=(a<<b))"},
{"x = a|b", "Stmt(x=(a|b))"},
{"x = a&b", "Stmt(x=(a&b))"},
{"x = a^b", "Stmt(x=(a^b))"},
{"x = a||b", "Stmt(x=(a||b))"},
{"x = a&&b", "Stmt(x=(a&&b))"},
{"x = !a", "Stmt(x=(!a))"},
{"x = delete a", "Stmt(x=(delete a))"},
{"x = a in b", "Stmt(x=(a in b))"},
{"x = a.replace(b, c)", "Stmt(x=((a.replace)(b, c)))"},
{"x &&= a", "Stmt(x&&=a)"},
{"x ||= a", "Stmt(x||=a)"},
{"x ??= a", "Stmt(x??=a)"},
{"class a extends async function(){}{}", "Decl(class a extends Decl(async function Params() Stmt({ })))"},
{"x = a?b:c=d", "Stmt(x=(a ? b : (c=d)))"},
{"implements = 0", "Stmt(implements=0)"},
{"interface = 0", "Stmt(interface=0)"},
{"let = 0", "Stmt(let=0)"},
{"(let [a] = 0)", "Stmt(((let[a])=0))"},
{"package = 0", "Stmt(package=0)"},
{"private = 0", "Stmt(private=0)"},
{"protected = 0", "Stmt(protected=0)"},
{"public = 0", "Stmt(public=0)"},
{"static = 0", "Stmt(static=0)"},
// expression to arrow function parameters
{"x = (a,b,c) => {a++}", "Stmt(x=(Params(Binding(a), Binding(b), Binding(c)) => Stmt({ Stmt(a++) })))"},
{"x = (a,b,...c) => {a++}", "Stmt(x=(Params(Binding(a), Binding(b), ...Binding(c)) => Stmt({ Stmt(a++) })))"},
{"x = ([a, ...b]) => {a++}", "Stmt(x=(Params(Binding([ Binding(a), ...Binding(b) ])) => Stmt({ Stmt(a++) })))"},
{"x = ([,a,]) => {a++}", "Stmt(x=(Params(Binding([ Binding(), Binding(a) ])) => Stmt({ Stmt(a++) })))"},
{"x = ({a}) => {a++}", "Stmt(x=(Params(Binding({ Binding(a) })) => Stmt({ Stmt(a++) })))"},
{"x = ({a:b, c:d}) => {a++}", "Stmt(x=(Params(Binding({ a: Binding(b), c: Binding(d) })) => Stmt({ Stmt(a++) })))"},
{"x = ({a:[b]}) => {a++}", "Stmt(x=(Params(Binding({ a: Binding([ Binding(b) ]) })) => Stmt({ Stmt(a++) })))"},
{"x = ({a=5}) => {a++}", "Stmt(x=(Params(Binding({ Binding(a = 5) })) => Stmt({ Stmt(a++) })))"},
{"x = ({...a}) => {a++}", "Stmt(x=(Params(Binding({ ...Binding(a) })) => Stmt({ Stmt(a++) })))"},
{"x = ([{...a}]) => {a++}", "Stmt(x=(Params(Binding([ Binding({ ...Binding(a) }) ])) => Stmt({ Stmt(a++) })))"},
{"x = ([{a: b}]) => {a++}", "Stmt(x=(Params(Binding([ Binding({ a: Binding(b) }) ])) => Stmt({ Stmt(a++) })))"},
{"x = (a = 5) => {a++}", "Stmt(x=(Params(Binding(a = 5)) => Stmt({ Stmt(a++) })))"},
{"x = ({a = 5}) => {a++}", "Stmt(x=(Params(Binding({ Binding(a = 5) })) => Stmt({ Stmt(a++) })))"},
// expression precedence
{"!!a", "Stmt(!(!a))"},
{"x = a.b.c", "Stmt(x=((a.b).c))"},
{"x = a+b+c", "Stmt(x=((a+b)+c))"},
{"x = a**b**c", "Stmt(x=(a**(b**c)))"},
{"a++ < b", "Stmt((a++)<b)"},
{"a&&b&&c", "Stmt((a&&b)&&c)"},
{"a||b||c", "Stmt((a||b)||c)"},
{"new new a(b)", "Stmt(new (new a(b)))"},
{"new super.a(b)", "Stmt(new (super.a)(b))"},
{"new new.target(a)", "Stmt(new (new.target)(a))"},
{"new import.meta(a)", "Stmt(new (import.meta)(a))"},
{"a(b)[c]", "Stmt((a(b))[c])"},
{"a[b]`tmpl`", "Stmt((a[b])`tmpl`)"},
{"a||b?c:d", "Stmt((a||b) ? c : d)"},
{"a??b?c:d", "Stmt((a??b) ? c : d)"},
{"a==b==c", "Stmt((a==b)==c)"},
{"new a?.b", "Stmt((new a)?.b)"},
{"new a++", "Stmt((new a)++)"},
{"new a--", "Stmt((new a)--)"},
{"a<<b<<c", "Stmt((a<<b)<<c)"},
{"a&b&c", "Stmt((a&b)&c)"},
{"a|b|c", "Stmt((a|b)|c)"},
{"a^b^c", "Stmt((a^b)^c)"},
{"a,b,c", "Stmt((a,b),c)"},
// regular expressions
{"/abc/", "Stmt(/abc/)"},
{"return /abc/;", "Stmt(return /abc/)"},
{"a/b/g", "Stmt((a/b)/g)"},
{"{}/1/g", "Stmt({ }) Stmt(/1/g)"},
{"i(0)/1/g", "Stmt(((i(0))/1)/g)"},
{"if(0)/1/g", "Stmt(if 0 Stmt(/1/g))"},
{"a.if(0)/1/g", "Stmt((((a.if)(0))/1)/g)"},
{"this/1/g", "Stmt((this/1)/g)"},
{"switch(a){case /1/g:}", "Stmt(switch a Clause(case /1/g))"},
{"(a+b)/1/g", "Stmt((((a+b))/1)/g)"},
{"f(); function foo() {} /42/i", "Stmt(f()) Decl(function foo Params() Stmt({ })) Stmt(/42/i)"},
{"x = function() {} /42/i", "Stmt(x=((Decl(function Params() Stmt({ }))/42)/i))"},
{"x = function foo() {} /42/i", "Stmt(x=((Decl(function foo Params() Stmt({ }))/42)/i))"},
{"x = /foo/", "Stmt(x=/foo/)"},
{"x = (/foo/)", "Stmt(x=(/foo/))"},
{"x = {a: /foo/}", "Stmt(x={a: /foo/})"},
{"x = (a) / foo", "Stmt(x=((a)/foo))"},
{"do { /foo/ } while (a)", "Stmt(do Stmt({ Stmt(/foo/) }) while a)"},
{"if (true) /foo/", "Stmt(if true Stmt(/foo/))"},
{"/abc/ ? /def/ : /geh/", "Stmt(/abc/ ? /def/ : /geh/)"},
{"yield * /abc/", "Stmt(yield*/abc/)"},
// variable reuse
{"var a; var a", "Decl(var Binding(a)) Decl(var Binding(a))"},
{"var a; {let a}", "Decl(var Binding(a)) Stmt({ Decl(let Binding(a)) })"},
{"{let a} var a", "Stmt({ Decl(let Binding(a)) }) Decl(var Binding(a))"},
{"function a(b,b){}", "Decl(function a Params(Binding(b), Binding(b)) Stmt({ }))"},
{"function a(b){var b}", "Decl(function a Params(Binding(b)) Stmt({ Decl(var Binding(b)) }))"},
{"a=function(b){var b}", "Stmt(a=Decl(function Params(Binding(b)) Stmt({ Decl(var Binding(b)) })))"},
{"a=function b(){var b}", "Stmt(a=Decl(function b Params() Stmt({ Decl(var Binding(b)) })))"},
{"a=function b(){let b}", "Stmt(a=Decl(function b Params() Stmt({ Decl(let Binding(b)) })))"},
{"a=>{var a}", "Stmt(Params(Binding(a)) => Stmt({ Decl(var Binding(a)) }))"},
{"var a;function a(){}", "Decl(var Binding(a)) Decl(function a Params() Stmt({ }))"},
{"try{}catch(a){var a}", "Stmt(try Stmt({ }) catch Binding(a) Stmt({ Decl(var Binding(a)) }))"},
// ASI
{"return a", "Stmt(return a)"},
{"return; a", "Stmt(return) Stmt(a)"},
{"return\na", "Stmt(return) Stmt(a)"},
{"return /*comment*/ a", "Stmt(return a)"},
{"return /*com\nment*/ a", "Stmt(return) Stmt(a)"},
{"return //comment\n a", "Stmt(return) Stmt(a)"},
{"a?.b\n`c`", "Stmt((a?.b)`c`)"},
{"() => { const v=6; x={v} }", "Stmt(Params() => Stmt({ Decl(const Binding(v = 6)) Stmt(x={v}) }))"},
{`([]=l=>{let{e}={e}})`, `Stmt(([]=(Params(Binding(l)) => Stmt({ Decl(let Binding({ Binding(e) } = {e})) }))))`}, // go-fuzz
}
for _, tt := range tests {
t.Run(tt.js, func(t *testing.T) {
ast, err := Parse(parse.NewInputString(tt.js))
if err != io.EOF {
test.Error(t, err)
}
test.String(t, ast.String(), tt.expected)
})
}
// coverage
for i := 0; ; i++ {
if OpPrec(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
for i := 0; ; i++ {
if DeclType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
}
func TestParseError(t *testing.T) {
var tests = []struct {
js string
err string
}{
{"{a", "unexpected EOF"},
{"if", "expected ( instead of EOF in if statement"},
{"if(a", "expected ) instead of EOF in if statement"},
{"if(a)let b", "unexpected b in expression"},
{"if(a)const b", "unexpected const in statement"},
{"with", "expected ( instead of EOF in with statement"},
{"with(a", "expected ) instead of EOF in with statement"},
{"do a++", "expected while instead of EOF in do-while statement"},
{"do a++ while", "unexpected while in expression"},
{"do a++; while", "expected ( instead of EOF in do-while statement"},
{"do a++; while(a", "expected ) instead of EOF in do-while statement"},
{"while", "expected ( instead of EOF in while statement"},
{"while(a", "expected ) instead of EOF in while statement"},
{"for", "expected ( instead of EOF in for statement"},
{"for(a", "expected in, of, or ; instead of EOF in for statement"},
{"for(a;a", "expected ; instead of EOF in for statement"},
{"for(a;a;a", "expected ) instead of EOF in for statement"},
{"for(var [a],b;", "unexpected ; in for statement"},
{"for(var [a]=5,{b};", "expected = instead of ; in var statement"},
{"for await", "expected ( instead of await in for statement"},
{"async function a(){ for await(a;", "expected of instead of ; in for statement"},
{"async function a(){ for await(a in", "expected of instead of in in for statement"},
{"for(var a of b", "expected ) instead of EOF in for statement"},
{"switch", "expected ( instead of EOF in switch statement"},
{"switch(a", "expected ) instead of EOF in switch statement"},
{"switch(a)", "expected { instead of EOF in switch statement"},
{"switch(a){bad:5}", "expected case or default instead of bad in switch statement"},
{"switch(a){case", "unexpected EOF in expression"},
{"switch(a){case a", "expected : instead of EOF in switch statement"},
{"switch(a){case a:", "unexpected EOF in switch statement"},
{"try", "expected { instead of EOF in try statement"},
{"try{", "unexpected EOF"},
{"try{}", "expected catch or finally instead of EOF in try statement"},
{"try{}catch(a", "expected ) instead of EOF in try-catch statement"},
{"try{}catch(a,", "expected ) instead of , in try-catch statement"},
{"try{}catch", "expected { instead of EOF in try-catch statement"},
{"try{}finally", "expected { instead of EOF in try-finally statement"},
{"function", "expected Identifier instead of EOF in function declaration"},
{"function(", "expected Identifier instead of ( in function declaration"},
{"!function", "expected Identifier or ( instead of EOF in function declaration"},
{"async function", "expected Identifier instead of EOF in function declaration"},
{"function a", "expected ( instead of EOF in function declaration"},
{"function a(b", "unexpected EOF in function declaration"},
{"function a(b,", "unexpected EOF in function declaration"},
{"function a(...b", "expected ) instead of EOF in function declaration"},
{"function a()", "expected { instead of EOF in function declaration"},
{"class", "expected Identifier instead of EOF in class declaration"},
{"class{", "expected Identifier instead of { in class declaration"},
{"!class", "expected { instead of EOF in class declaration"},
{"class A", "expected { instead of EOF in class declaration"},
{"class A{", "unexpected EOF in class declaration"},
{"class A extends a b {}", "expected { instead of b in class declaration"},
{"class A{+", "expected Identifier, String, Numeric, or [ instead of + in method definition"},
{"class A{[a", "expected ] instead of EOF in method definition"},
{"var [...a", "expected ] instead of EOF in array binding pattern"},
{"var [a", "expected , or ] instead of EOF in array binding pattern"},
{"var [a]", "expected = instead of EOF in var statement"},
{"var {[a", "expected ] instead of EOF in object binding pattern"},
{"var {+", "expected Identifier, String, Numeric, or [ instead of + in object binding pattern"},
{"var {a", "expected , or } instead of EOF in object binding pattern"},
{"var {...a", "expected } instead of EOF in object binding pattern"},
{"var {a}", "expected = instead of EOF in var statement"},
{"var 0", "unexpected 0 in binding"},
{"x={", "expected } instead of EOF in object literal"},
{"x={[a", "expected ] instead of EOF in object literal"},
{"x={[a]", "expected : or ( instead of EOF in object literal"},
{"x={+", "expected Identifier, String, Numeric, or [ instead of + in object literal"},
{"x={async\na", "unexpected a in object literal"},
{"class a extends ||", "unexpected || in expression"},
{"class a extends =", "unexpected = in expression"},
{"class a extends ?", "unexpected ? in expression"},
{"class a extends =>", "unexpected => in expression"},
{"x=a?b", "expected : instead of EOF in conditional expression"},
{"x=(a", "unexpected EOF in expression"},
{"x+(a", "expected ) instead of EOF in expression"},
{"x={a", "unexpected EOF in object literal"},
{"x=a[b", "expected ] instead of EOF in index expression"},
{"x=async a", "expected => instead of EOF in arrow function"},
{"x=async (a", "unexpected EOF in expression"},
{"x=async (a,", "unexpected EOF in expression"},
{"x=async function", "expected Identifier or ( instead of EOF in function declaration"},
{"x=async function *", "expected Identifier or ( instead of EOF in function declaration"},
{"x=async function a", "expected ( instead of EOF in function declaration"},
{"x=?.?.b", "unexpected ?. in expression"},
{"x=a?.?.b", "expected Identifier, (, [, or Template instead of ?. in optional chaining expression"},
{"x=a?..b", "expected Identifier, (, [, or Template instead of . in optional chaining expression"},
{"x=a?.[b", "expected ] instead of EOF in optional chaining expression"},
{"`tmp${", "unexpected EOF in expression"},
{"`tmp${x", "expected Template instead of EOF in template literal"},
{"`tmpl` x `tmpl`", "unexpected x in expression"},
{"x=5=>", "unexpected => in expression"},
{"x=new.bad", "expected target instead of bad in new.target expression"},
{"x=import.bad", "expected meta instead of bad in import.meta expression"},
{"x=super", "expected [, (, or . instead of EOF in super expression"},
{"x=super(a", "expected ) instead of EOF in arguments"},
{"x=super[a", "expected ] instead of EOF in index expression"},
{"x=super.", "expected Identifier instead of EOF in dot expression"},
{"x=new super(b)", "expected [ or . instead of ( in super expression"},
{"x=import", "expected ( instead of EOF in import expression"},
{"x=import(5", "expected ) instead of EOF in arguments"},
{"x=new import(b)", "unexpected ( in expression"},
{"import", "expected String, Identifier, *, or { instead of EOF in import statement"},
{"import *", "expected as instead of EOF in import statement"},
{"import * as", "expected Identifier instead of EOF in import statement"},
{"import {", "expected } instead of EOF in import statement"},
{"import {yield", "expected } instead of EOF in import statement"},
{"import {yield as", "expected Identifier instead of EOF in import statement"},
{"import {yield,", "expected } instead of EOF in import statement"},
{"import yield", "expected from instead of EOF in import statement"},
{"import yield from", "expected String instead of EOF in import statement"},
{"export", "expected *, {, var, let, const, function, async, class, or default instead of EOF in export statement"},
{"export *", "expected from instead of EOF in export statement"},
{"export * as", "expected Identifier instead of EOF in export statement"},
{"export * as if", "expected from instead of EOF in export statement"},
{"export {", "expected } instead of EOF in export statement"},
{"export {yield", "expected } instead of EOF in export statement"},
{"export {yield,", "expected } instead of EOF in export statement"},
{"export {yield as", "expected Identifier instead of EOF in export statement"},
{"export {} from", "expected String instead of EOF in export statement"},
{"export {} from", "expected String instead of EOF in export statement"},
{"export async", "expected function instead of EOF in export statement"},
// no declarations
{"if(a) function f(){}", "unexpected function in statement"},
{"if(a) async function f(){}", "unexpected async in statement"},
{"if(a) class c{}", "unexpected class in statement"},
// yield, async, await
{"yield a = 5", "unexpected a in expression"},
{"function*a() { yield: var a", "unexpected : in expression"},
{"function*a() { x = b + yield c", "unexpected yield in expression"},
{"function a(b = yield c){}", "unexpected c in function declaration"},
{"function*a(){ (yield) => yield }", "unexpected => in expression"},
{"function*a(){ (yield=5) => yield }", "unexpected = in expression"},
{"function*a(){ (...yield) => yield }", "unexpected yield in arrow function"},
{"x = await\n=> a++", "unexpected => in expression"},
{"x=async (await,", "unexpected EOF in expression"},
{"async function a() { class a extends await", "unexpected await in expression"},
{"async function a() { await: var a", "unexpected : in expression"},
{"async function a() { let await", "unexpected await in binding"},
{"async function a() { let\nawait", "unexpected await in binding"},
{"async function a() { x = new await c", "unexpected await in expression"},
{"async function a() { x = await =>", "unexpected => in expression"},
{"async function a(){ (await) => await }", "unexpected ) in expression"},
{"async function a(){ (await=5) => await }", "unexpected = in expression"},
{"async function a(){ (...await) => await }", "unexpected await in arrow function"},
{"async+a b", "unexpected b in expression"},
{"(async\nfunction(){})", "unexpected function in expression"},
{"a + async b", "unexpected b in expression"},
{"async await => 5", "unexpected await in arrow function"},
// specific cases
{"{a, if: b, do(){}, ...d}", "unexpected if in expression"}, // block stmt
{"let {if = 5}", "expected : instead of = in object binding pattern"},
{"let {...}", "expected Identifier instead of } in object binding pattern"},
{"let {...[]}", "expected Identifier instead of [ in object binding pattern"},
{"let {...{}}", "expected Identifier instead of { in object binding pattern"},
{"for", "expected ( instead of EOF in for statement"},
{"for b", "expected ( instead of b in for statement"},
{"for (a b)", "expected in, of, or ; instead of b in for statement"},
{"for (var a in b;) {}", "expected ) instead of ; in for statement"},
{"for (var a,b in c) {}", "unexpected in in for statement"},
{"for (var a,b of c) {}", "unexpected of in for statement"},
{"if (a) 1 else 3", "unexpected else in expression"},
{"x = [...]", "unexpected ] in expression"},
{"x = {...}", "unexpected } in expression"},
{"let\nawait 0", "unexpected 0 in let declaration"},
{"const\nawait 0", "unexpected 0 in const declaration"},
{"var\nawait 0", "unexpected 0 in var statement"},
// expression to arrow function parameters
{"x = ()", "expected => instead of EOF in arrow function"},
{"x = [x] => a", "unexpected => in expression"},
{"x = ((x)) => a", "unexpected => in expression"},
{"x = ([...x, y]) => a", "unexpected => in expression"},
{"x = ({...x, y}) => a", "unexpected => in expression"},
{"x = ({b(){}}) => a", "unexpected => in expression"},
{"x = (a, b, ...c)", "expected => instead of EOF in arrow function"},
{"x = (a+b) =>", "unexpected => in expression"},
{"x = ([...a, b]) =>", "unexpected => in expression"},
{"x = ([...5]) =>", "unexpected => in expression"},
{"x = ([5]) =>", "unexpected => in expression"},
{"x = ({...a, b}) =>", "unexpected => in expression"},
{"x = ({...5}) =>", "unexpected => in expression"},
{"x = ({5: 5}) =>", "unexpected => in expression"},
{"x = ({[4+5]: 5}) =>", "unexpected => in expression"},
// expression precedence
{"x = a + yield b", "unexpected b in expression"},
{"a??b||c", "unexpected || in expression"},
{"a??b&&c", "unexpected && in expression"},
{"a||b??c", "unexpected ?? in expression"},
{"a&&b??c", "unexpected ?? in expression"},
{"x = a++--", "unexpected -- in expression"},
{"x = a\n++", "unexpected EOF in expression"},
{"x = a++?", "unexpected EOF in expression"},
{"a+b =", "unexpected = in expression"},
{"!a**b", "unexpected ** in expression"},
{"new !a", "unexpected ! in expression"},
{"new +a", "unexpected + in expression"},
{"new -a", "unexpected - in expression"},
{"new ++a", "unexpected ++ in expression"},
{"new --a", "unexpected -- in expression"},
{"a=>{return a} < b", "unexpected < in expression"},
{"a=>{return a} == b", "unexpected == in expression"},
{"a=>{return a} . b", "unexpected . in expression"},
{"a=>{return a} (", "unexpected ( in expression"},
{"a=>{return a} [", "unexpected [ in expression"},
{"a=>{return a} `", "unexpected ` in expression"},
{"a=>{return a} ++", "unexpected ++ in expression"},
{"a=>{return a} --", "unexpected -- in expression"},
{"a=>{return a} * b", "unexpected * in expression"},
{"a=>{return a} + b", "unexpected + in expression"},
{"a=>{return a} << b", "unexpected << in expression"},
{"a=>{return a} & b", "unexpected & in expression"},
{"a=>{return a} | b", "unexpected | in expression"},
{"a=>{return a} ^ b", "unexpected ^ in expression"},
{"a=>{return a} ? b", "unexpected ? in expression"},
{"a=>{return a} => b=>b", "unexpected => in expression"},
{"class a extends b=>b", "expected { instead of => in class declaration"},
// regular expressions
{"x = x / foo /", "unexpected EOF in expression"},
{"bar (true) /foo/", "unexpected EOF in expression"},
{"yield /abc/", "unexpected EOF in expression"},
// variable reuse
{"let a; var a", "identifier a has already been declared"},
{"let a; {var a}", "identifier a has already been declared"},
{"{let a; {var a}}", "identifier a has already been declared"},
{"var a; let a", "identifier a has already been declared"},
{"{var a} let a", "identifier a has already been declared"},
{"var a; const a", "identifier a has already been declared"},
{"var a; class a{}", "identifier a has already been declared"},
{"function a(b){let b}", "identifier b has already been declared"},
{"a=function(b){let b}", "identifier b has already been declared"},
{"a=>{let a}", "identifier a has already been declared"},
{"let a;function a(){}", "identifier a has already been declared"},
{"try{}catch(a){let a}", "identifier a has already been declared"},
{"let {a, a}", "identifier a has already been declared"},
{"let {a, ...a}", "identifier a has already been declared"},
{"for(let a in [0,1,2]){var a = 5}", "identifier a has already been declared"},
{"for(let a=0; a<10; a++){var a = 5}", "identifier a has already been declared"},
// other
{"\x00", "unexpected 0x00"},
{"@", "unexpected @"},
{"\u200F", "unexpected U+200F"},
{"\u2010", "unexpected \u2010"},
{"a=\u2010", "unexpected \u2010 in expression"},
{"/", "unexpected EOF or newline in regular expression"},
{"({...[]})=>a", "unexpected => in expression"}, // go-fuzz
}
for _, tt := range tests {
t.Run(tt.js, func(t *testing.T) {
_, err := Parse(parse.NewInputString(tt.js))
test.That(t, err != io.EOF && err != nil)
e := err.Error()
if len(tt.err) < len(err.Error()) {
e = e[:len(tt.err)]
}
test.String(t, e, tt.err)
})
}
}
type ScopeVars struct {
bound, uses string
scopes int
refs map[*Var]int
}
func NewScopeVars() *ScopeVars {
return &ScopeVars{
refs: map[*Var]int{},
}
}
func (sv *ScopeVars) String() string {
return "bound:" + sv.bound + " uses:" + sv.uses
}
func (sv *ScopeVars) Ref(v *Var) int {
if ref, ok := sv.refs[v]; ok {
return ref
}
sv.refs[v] = len(sv.refs) + 1
return len(sv.refs)
}
func (sv *ScopeVars) AddScope(scope Scope) {
if sv.scopes != 0 {
sv.bound += "/"
sv.uses += "/"
}
sv.scopes++
bounds := []string{}
for _, v := range scope.Declared {
bounds = append(bounds, fmt.Sprintf("%s=%d", string(v.Data), sv.Ref(v)))
}
sv.bound += strings.Join(bounds, ",")
uses := []string{}
for _, v := range scope.Undeclared {
links := ""
for v.Link != nil {
v = v.Link
links += "*"
}
uses = append(uses, fmt.Sprintf("%s=%d%s", string(v.Data), sv.Ref(v), links))
}
sv.uses += strings.Join(uses, ",")
}
func (sv *ScopeVars) AddExpr(iexpr IExpr) {
switch expr := iexpr.(type) {
case *FuncDecl:
sv.AddScope(expr.Body.Scope)
for _, item := range expr.Params.List {
if item.Binding != nil {
sv.AddBinding(item.Binding)
}
if item.Default != nil {
sv.AddExpr(item.Default)
}
}
if expr.Params.Rest != nil {
sv.AddBinding(expr.Params.Rest)
}
for _, item := range expr.Body.List {
sv.AddStmt(item)
}
case *ClassDecl:
for _, method := range expr.Methods {
sv.AddScope(method.Body.Scope)
}
case *ArrowFunc:
sv.AddScope(expr.Body.Scope)
for _, item := range expr.Params.List {
if item.Binding != nil {
sv.AddBinding(item.Binding)
}
if item.Default != nil {
sv.AddExpr(item.Default)
}
}
if expr.Params.Rest != nil {
sv.AddBinding(expr.Params.Rest)
}
for _, item := range expr.Body.List {
sv.AddStmt(item)
}
case *CondExpr:
sv.AddExpr(expr.Cond)
sv.AddExpr(expr.X)
sv.AddExpr(expr.Y)
case *UnaryExpr:
sv.AddExpr(expr.X)
case *BinaryExpr:
sv.AddExpr(expr.X)
sv.AddExpr(expr.Y)
case *GroupExpr:
sv.AddExpr(expr.X)
}
}
func (sv *ScopeVars) AddBinding(ibinding IBinding) {
switch binding := ibinding.(type) {
case *BindingArray:
for _, item := range binding.List {
if item.Binding != nil {
sv.AddBinding(item.Binding)
}
if item.Default != nil {
sv.AddExpr(item.Default)
}
}
if binding.Rest != nil {
sv.AddBinding(binding.Rest)
}
case *BindingObject:
for _, item := range binding.List {
if item.Key.IsComputed() {
sv.AddExpr(item.Key.Computed)
}
if item.Value.Binding != nil {
sv.AddBinding(item.Value.Binding)
}
if item.Value.Default != nil {
sv.AddExpr(item.Value.Default)
}
}
}
}
func (sv *ScopeVars) AddStmt(istmt IStmt) {
switch stmt := istmt.(type) {
case *BlockStmt:
sv.AddScope(stmt.Scope)
for _, item := range stmt.List {
sv.AddStmt(item)
}
case *FuncDecl:
sv.AddScope(stmt.Body.Scope)
for _, item := range stmt.Params.List {
if item.Binding != nil {
sv.AddBinding(item.Binding)
}
if item.Default != nil {
sv.AddExpr(item.Default)
}
}
if stmt.Params.Rest != nil {
sv.AddBinding(stmt.Params.Rest)
}
for _, item := range stmt.Body.List {
sv.AddStmt(item)
}
case *ClassDecl:
for _, method := range stmt.Methods {
sv.AddScope(method.Body.Scope)
}
case *ReturnStmt:
sv.AddExpr(stmt.Value)
case *ThrowStmt:
sv.AddExpr(stmt.Value)
case *ForStmt:
sv.AddStmt(stmt.Body)
case *ForInStmt:
sv.AddStmt(stmt.Body)
case *ForOfStmt:
sv.AddStmt(stmt.Body)
case *IfStmt:
sv.AddStmt(stmt.Body)
if stmt.Else != nil {
sv.AddStmt(stmt.Else)
}
case *TryStmt:
if 0 < len(stmt.Body.List) {
sv.AddStmt(stmt.Body)
}
if stmt.Catch != nil {
sv.AddStmt(stmt.Catch)
}
if stmt.Finally != nil {
sv.AddStmt(stmt.Finally)
}
case *VarDecl:
for _, item := range stmt.List {
if item.Default != nil {
sv.AddExpr(item.Default)
}
}
case *ExprStmt:
sv.AddExpr(stmt.Value)
}
}
func TestParseScope(t *testing.T) {
// vars registers all bound and unbound variables per scope. Unbound variables are not defined in that particular scope and are defined in another scope (parent, global, child of a parent, ...). Bound variables are variables that are defined in this scope. Each scope is separated by /, and the variables are separated by commas. Each variable is assigned a unique ID (sort by first bounded than unbounded per scope) in order to make sure which identifiers refer to the same variable.
// var and function declarations are function-scoped
// const, let, and class declarations are block-scoped
var tests = []struct {
js string
bound, uses string
}{
{"a; a", "", "a=1"},
{"a;{a;{a}}", "//", "a=1/a=1/a=1*"},
{"var a; b", "a=1", "b=2"},
{"var {a:b, c=d, ...e} = z;", "b=1,c=2,e=3", "d=4,z=5"},
{"var [a, b=c, ...d] = z;", "a=1,b=2,d=3", "c=4,z=5"},
{"x={a:b, c=d, ...e};", "", "x=1,b=2,c=3,d=4,e=5"},
{"x=[a, b=c, ...d];", "", "x=1,a=2,b=3,c=4,d=5"},
{"yield = 5", "", "yield=1"},
{"await = 5", "", "await=1"},
{"function a(b,c){var d; e = 5; a}", "a=1/b=3,c=4,d=5", "e=2/e=2,a=1"},
{"function a(b,c=b){}", "a=1/b=2,c=3", "/"},
{"function a(b=c,c){}", "a=1/b=3,c=4", "c=2/c=2"},
{"function a(b=c){var c}", "a=1/b=3,c=4", "c=2/c=2"},
{"function a(b){var b}", "a=1/b=2", "/"},
{"function a(b,b){}", "a=1/b=2", "/"},
{"!function a(b,c){var d; e = 5; a}", "/a=2,b=3,c=4,d=5", "e=1/e=1"},
{"a=function(b,c=b){}", "/b=2,c=3", "a=1/"},
{"a=function(b=c,c){}", "/b=3,c=4", "a=1,c=2/c=2"},
{"a=function(b=c){var c}", "/b=3,c=4", "a=1,c=2/c=2"},
{"a=function(b){var b}", "/b=2", "a=1/"},
{"a=function(b,b){}", "/b=2", "a=1/"},
{"class a{b(){}}", "a=1/", "/"}, // classes are not tracked
{"!class a{b(){}}", "/", "/"},
{"a => a%5", "/a=1", "/"},
{"a => a%b", "/a=2", "b=1/b=1"},
{"var a;a => a%5", "a=1/a=2", "/"},
{"(a) + (a)", "", "a=1"},
{"(a,b)", "", "a=1,b=2"},
{"(a,b) + (a,b)", "", "a=1,b=2"},
{"(a) + (a => a%5)", "/a=2", "a=1/"},
{"(a=b) => {var c; d = 5}", "/a=3,c=4", "b=1,d=2/b=1,d=2"},
{"(a,b=a) => {}", "/a=1,b=2", "/"},
{"(a=b,b)=>{}", "/a=2,b=3", "b=1/b=1"},
{"a=>{var a}", "/a=1", "/"},
{"(a,a)=>{}", "/a=1", "/"},
{"(a=b) => {var b}", "/a=2,b=3", "b=1/b=1"},
{"({[a+b]:c}) => {}", "/c=3", "a=1,b=2/a=1,b=2"},
{"({a:b, c=d, ...e}=f) => 5", "/b=3,c=4,e=5", "d=1,f=2/d=1,f=2"},
{"([a, b=c, ...d]=e) => 5", "/a=3,b=4,d=5", "c=1,e=2/c=1,e=2"},
{"(a) + ((b,c) => {var d; e = 5; return e})", "/b=3,c=4,d=5", "a=1,e=2/e=2"},
{"(a) + ((a,b) => {var c; d = 5; return d})", "/a=3,b=4,c=5", "a=1,d=2/d=2"},
{"{(a) + ((a,b) => {var c; d = 5; return d})}", "//a=3,b=4,c=5", "a=1,d=2/a=1,d=2/d=2"},
{"(a=(b=>b/a)) => a", "/a=1/b=2", "//a=1*"},
{"(a=(b=>b/c)) => a", "/a=2/b=3", "c=1/c=1/c=1"},
{"(a=(function b(){})) => a", "/a=1/b=2", "//"},
{"label: a", "", "a=1"},
{"yield => yield%5", "/yield=1", "/"},
{"await => await%5", "/await=1", "/"},
{"function*a(){b => yield%5}", "a=1//b=3", "yield=2/yield=2/yield=2"},
{"async function a(){b => await%5}", "a=1//b=3", "await=2/await=2/await=2"},
{"let a; {let b = a}", "a=1/b=2", "/a=1"},
{"let a; {var b}", "a=1,b=2/", "/b=2"}, // may remove b from uses
{"let a; {var b = a}", "a=1,b=2/", "/b=2,a=1"},
{"let a; {class b{}}", "a=1/b=2", "/"},
{"a = 5; var a;", "a=1", ""},
{"a = 5; let a;", "a=1", ""},
{"a = 5; {var a}", "a=1/", "/a=1"},
{"a = 5; {let a}", "/a=2", "a=1/"},
{"{a = 5} var a", "a=1/", "/a=1"},
{"{a = 5} let a", "a=1/", "/a=1"},
{"var a; {a = 5}", "a=1/", "/a=1"},
{"var a; {var a}", "a=1/", "/a=1"},
{"var a; {let a}", "a=1/a=2", "/"},
{"let a; {a = 5}", "a=1/", "/a=1"},
{"{var a} a = 5", "a=1/", "/a=1"},
{"{let a} a = 5", "/a=2", "a=1/"},
{"!function(){throw new Error()}", "/", "Error=1/Error=1"},
{"!function(){return a}", "/", "a=1/a=1"},
{"!function(){return a};var a;", "a=1/", "/a=1"},
{"!function(){return a};if(5){var a}", "a=1//", "/a=1/a=1"},
{"try{}catch(a){var a}", "a=1/a=2", "/a=1"},
{"try{}catch(a){let b; c}", "/a=2,b=3", "c=1/c=1"},
{"try{}catch(a){var b; c}", "b=1/a=3", "c=2/b=1,c=2"},
{"var a;try{}catch(a){a}", "a=1/a=2", "/"},
{"var a;try{}catch(a){var a}", "a=1/a=2", "/a=1"},
{"var a;try{}catch(b){var a}", "a=1/b=2", "/a=1"},
{"function r(o){function l(t){if(!z[t]){if(!o[t]);}}}", "r=1/o=3,l=4/t=5/", "z=2/z=2/z=2,o=3/o=3*,t=5"},
{"function a(){var name;{var name}}", "a=1/name=2/", "//name=2"}, // may remove name from uses
{"function a(){var name;{function name(){}}}", "a=1/name=2//", "//name=2/"}, // may remove name from uses
{"function a(){var name;{var name=7}}", "a=1/name=2/", "//name=2"},
{"!function(){a};!function(){a};var a", "a=1//", "/a=1/a=1"},
{"!function(){var a;!function(){a;var a}}", "/a=1/a=2", "//"},
{"!function(){var a;!function(){!function(){a}}}", "/a=1//", "//a=1/a=1*"},
{"!function(){var a;!function(){a;!function(){a}}}", "/a=1//", "//a=1/a=1*"},
{"!function(){var b;{(T=x),T}{var T}}", "/b=2,T=3//", "x=1/x=1/x=1,T=3/T=3"},
{"var T;!function(){var b;{(T=x),T}{var T}}", "T=1/b=3,T=4//", "x=2/x=2/x=2,T=4/T=4"},
{"!function(){let a=b,b=c,c=d,d=e,e=f,f=g,g=h,h=a,j;for(let i=0;;)j=4;}", "/a=1,b=2,c=3,d=4,e=5,f=6,g=7,h=8,j=9/i=10", "//j=9"},
{"{a} {a} var a", "a=1//", "/a=1/a=1"}, // second block must add a new var in case the block contains a var decl
{"(a),(a)", "", "a=1"}, // second parens could have been arrow function, so must have added new var
{"var a,b,c;(a = b[c])", "a=1,b=2,c=3", ""}, // parens could have been arrow function, so must have added new var
{"!function(a){var b,c;return b?(c=function(){return[a];a.dispatch()},c):t}", "/a=2,b=3,c=4/", "t=1/t=1/a=2*"},
{"(...{a=function(){return [b]}}) => 5", "/a=2/", "b=1/b=1/b=1"},
{"(...[a=function(){return [b]}]) => 5", "/a=2/", "b=1/b=1/b=1"},
{`a=>{for(let b of c){b,a;{var d}}}`, "/a=2,d=3/b=4/", "c=1/c=1/c=1,a=2,d=3/d=3"},
{`var a;{let b;{var a}}`, "a=1/b=2/", "/a=1/a=1"},
{`for(let b of c){let b;{b}}`, "/b=2,b=3/", "c=1/c=1/b=3"},
{`for(var b of c){let b;{b}}`, "b=1/b=3/", "c=2/b=1,c=2/b=3"},
{`for(var b of c){var b;{b}}`, "b=1//", "c=2/b=1,c=2/b=1"},
}
for _, tt := range tests {
t.Run(tt.js, func(t *testing.T) {
ast, err := Parse(parse.NewInputString(tt.js))
if err != io.EOF {
test.Error(t, err)
}
vars := NewScopeVars()
vars.AddScope(ast.Scope)
for _, istmt := range ast.List {
vars.AddStmt(istmt)
}
test.String(t, vars.String(), "bound:"+tt.bound+" uses:"+tt.uses)
})
}
}
func TestScope(t *testing.T) {
js := "let a,b; b = 5; var c; {d}{{d}}"
ast, err := Parse(parse.NewInputString(js))
if err != io.EOF {
test.Error(t, err)
}
scope := ast.Scope
// test output
test.T(t, scope.String(), "Scope{Declared: [Var{LexicalDecl a 0 1}, Var{LexicalDecl b 0 2}, Var{VariableDecl c 0 1}], Undeclared: [Var{NoDecl d 0 2}]}")
// test sort
sort.Sort(VarsByUses(scope.Declared))
test.T(t, scope.String(), "Scope{Declared: [Var{LexicalDecl b 0 2}, Var{LexicalDecl a 0 1}, Var{VariableDecl c 0 1}], Undeclared: [Var{NoDecl d 0 2}]}")
// test variable link
test.T(t, ast.List[3].(*BlockStmt).Scope.String(), "Scope{Declared: [], Undeclared: [Var{NoDecl d 0 2}]}")
test.T(t, ast.List[4].(*BlockStmt).Scope.String(), "Scope{Declared: [], Undeclared: [Var{NoDecl d 0 2}]}")
test.T(t, ast.List[4].(*BlockStmt).List[0].(*BlockStmt).Scope.String(), "Scope{Declared: [], Undeclared: [Var{NoDecl d 1 2}]}")
}
func TestParseInputError(t *testing.T) {
_, err := Parse(parse.NewInput(test.NewErrorReader(0)))
test.T(t, err, test.ErrPlain)
_, err = Parse(parse.NewInput(test.NewErrorReader(1)))
test.T(t, err, test.ErrPlain)
}
parse-2.5.21/js/table.go 0000664 0000000 0000000 00000006750 14117006534 0014756 0 ustar 00root root 0000000 0000000 package js
import "strconv"
// OpPrec is the operator precedence
type OpPrec int
// OpPrec values.
const (
OpExpr OpPrec = iota // a,b
OpAssign // a?b:c, yield x, ()=>x, async ()=>x, a=b, a+=b, ...
OpCoalesce // a??b
OpOr // a||b
OpAnd // a&&b
OpBitOr // a|b
OpBitXor // a^b
OpBitAnd // a&b
OpEquals // a==b, a!=b, a===b, a!==b
OpCompare // a<b, a>b, a<=b, a>=b, a instanceof b, a in b
OpShift // a<<b, a>>b, a>>>b
OpAdd // a+b, a-b
OpMul // a*b, a/b, a%b
OpExp // a**b
OpUnary // ++x, --x, delete x, void x, typeof x, +x, -x, ~x, !x, await x
OpUpdate // x++, x--
OpLHS // CallExpr/OptChainExpr or NewExpr
OpCall // a?.b, a(b), super(a), import(a)
OpNew // new a
OpMember // a[b], a.b, a`b`, super[x], super.x, new.target, import.meta, new a(b)
OpPrimary // literal, function, class, parenthesized
)
func (prec OpPrec) String() string {
switch prec {
case OpExpr:
return "OpExpr"
case OpAssign:
return "OpAssign"
case OpCoalesce:
return "OpCoalesce"
case OpOr:
return "OpOr"
case OpAnd:
return "OpAnd"
case OpBitOr:
return "OpBitOr"
case OpBitXor:
return "OpBitXor"
case OpBitAnd:
return "OpBitAnd"
case OpEquals:
return "OpEquals"
case OpCompare:
return "OpCompare"
case OpShift:
return "OpShift"
case OpAdd:
return "OAdd"
case OpMul:
return "OpMul"
case OpExp:
return "OpExp"
case OpUnary:
return "OpUnary"
case OpUpdate:
return "OpUpdate"
case OpLHS:
return "OpLHS"
case OpCall:
return "OpCall"
case OpNew:
return "OpNew"
case OpMember:
return "OpMember"
case OpPrimary:
return "OpPrimary"
}
return "Invalid(" + strconv.Itoa(int(prec)) + ")"
}
// Keywords is a map of reserved, strict, and other keywords
var Keywords = map[string]TokenType{
// reserved
"await": AwaitToken,
"break": BreakToken,
"case": CaseToken,
"catch": CatchToken,
"class": ClassToken,
"const": ConstToken,
"continue": ContinueToken,
"debugger": DebuggerToken,
"default": DefaultToken,
"delete": DeleteToken,
"do": DoToken,
"else": ElseToken,
"enum": EnumToken,
"export": ExportToken,
"extends": ExtendsToken,
"false": FalseToken,
"finally": FinallyToken,
"for": ForToken,
"function": FunctionToken,
"if": IfToken,
"import": ImportToken,
"in": InToken,
"instanceof": InstanceofToken,
"new": NewToken,
"null": NullToken,
"return": ReturnToken,
"super": SuperToken,
"switch": SwitchToken,
"this": ThisToken,
"throw": ThrowToken,
"true": TrueToken,
"try": TryToken,
"typeof": TypeofToken,
"var": VarToken,
"void": VoidToken,
"while": WhileToken,
"with": WithToken,
"yield": YieldToken,
// strict mode
"let": LetToken,
"static": StaticToken,
"implements": ImplementsToken,
"interface": InterfaceToken,
"package": PackageToken,
"private": PrivateToken,
"protected": ProtectedToken,
"public": PublicToken,
// extra
"as": AsToken,
"async": AsyncToken,
"from": FromToken,
"get": GetToken,
"meta": MetaToken,
"of": OfToken,
"set": SetToken,
"target": TargetToken,
}
parse-2.5.21/js/tokentype.go 0000664 0000000 0000000 00000020533 14117006534 0015704 0 ustar 00root root 0000000 0000000 package js
import "strconv"
// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint16 // from LSB to MSB: 8 bits for tokens per category, 1 bit for numeric, 1 bit for punctuator, 1 bit for operator, 1 bit for identifier, 4 bits unused
// TokenType values.
const (
ErrorToken TokenType = iota // extra token when errors occur
WhitespaceToken
LineTerminatorToken // \r \n \r\n
CommentToken
CommentLineTerminatorToken
StringToken
TemplateToken
TemplateStartToken
TemplateMiddleToken
TemplateEndToken
RegExpToken
PrivateIdentifierToken
)
// Numeric token values.
const (
NumericToken TokenType = 0x0100 + iota
DecimalToken
BinaryToken
OctalToken
HexadecimalToken
BigIntToken
)
// Punctuator token values.
const (
PunctuatorToken TokenType = 0x0200 + iota
OpenBraceToken // {
CloseBraceToken // }
OpenParenToken // (
CloseParenToken // )
OpenBracketToken // [
CloseBracketToken // ]
DotToken // .
SemicolonToken // ;
CommaToken // ,
QuestionToken // ?
ColonToken // :
ArrowToken // =>
EllipsisToken // ...
)
// Operator token values.
const (
OperatorToken TokenType = 0x0600 + iota
EqToken // =
EqEqToken // ==
EqEqEqToken // ===
NotToken // !
NotEqToken // !=
NotEqEqToken // !==
LtToken // <
LtEqToken // <=
LtLtToken // <<
LtLtEqToken // <<=
GtToken // >
GtEqToken // >=
GtGtToken // >>
GtGtEqToken // >>=
GtGtGtToken // >>>
GtGtGtEqToken // >>>=
AddToken // +
AddEqToken // +=
IncrToken // ++
SubToken // -
SubEqToken // -=
DecrToken // --
MulToken // *
MulEqToken // *=
ExpToken // **
ExpEqToken // **=
DivToken // /
DivEqToken // /=
ModToken // %
ModEqToken // %=
BitAndToken // &
BitOrToken // |
BitXorToken // ^
BitNotToken // ~
BitAndEqToken // &=
BitOrEqToken // |=
BitXorEqToken // ^=
AndToken // &&
OrToken // ||
NullishToken // ??
AndEqToken // &&=
OrEqToken // ||=
NullishEqToken // ??=
OptChainToken // ?.
// unused in lexer
PosToken // +a
NegToken // -a
PreIncrToken // ++a
PreDecrToken // --a
PostIncrToken // a++
PostDecrToken // a--
)
// Reserved token values.
const (
ReservedToken TokenType = 0x0800 + iota
AwaitToken
BreakToken
CaseToken
CatchToken
ClassToken
ConstToken
ContinueToken
DebuggerToken
DefaultToken
DeleteToken
DoToken
ElseToken
EnumToken
ExportToken
ExtendsToken
FalseToken
FinallyToken
ForToken
FunctionToken
IfToken
ImportToken
InToken
InstanceofToken
NewToken
NullToken
ReturnToken
SuperToken
SwitchToken
ThisToken
ThrowToken
TrueToken
TryToken
TypeofToken
YieldToken
VarToken
VoidToken
WhileToken
WithToken
)
// Identifier token values.
const (
IdentifierToken TokenType = 0x1000 + iota
AsToken
AsyncToken
FromToken
GetToken
ImplementsToken
InterfaceToken
LetToken
MetaToken
OfToken
PackageToken
PrivateToken
ProtectedToken
PublicToken
SetToken
StaticToken
TargetToken
)
// IsNumeric return true if token is numeric.
func IsNumeric(tt TokenType) bool {
return tt&0x0100 != 0
}
// IsPunctuator return true if token is a punctuator.
func IsPunctuator(tt TokenType) bool {
return tt&0x0200 != 0
}
// IsOperator return true if token is an operator.
func IsOperator(tt TokenType) bool {
return tt&0x0400 != 0
}
// IsIdentifierName matches IdentifierName, i.e. any identifier
func IsIdentifierName(tt TokenType) bool {
return tt&0x1800 != 0
}
// IsReservedWord matches ReservedWord
func IsReservedWord(tt TokenType) bool {
return tt&0x0800 != 0
}
// IsIdentifier matches Identifier, i.e. IdentifierName but not ReservedWord. Does not match yield or await.
func IsIdentifier(tt TokenType) bool {
return tt&0x1000 != 0
}
func (tt TokenType) String() string {
s := tt.Bytes()
if s == nil {
return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}
return string(s)
}
var operatorBytes = [][]byte{
[]byte("Operator"),
[]byte("="),
[]byte("=="),
[]byte("==="),
[]byte("!"),
[]byte("!="),
[]byte("!=="),
[]byte("<"),
[]byte("<="),
[]byte("<<"),
[]byte("<<="),
[]byte(">"),
[]byte(">="),
[]byte(">>"),
[]byte(">>="),
[]byte(">>>"),
[]byte(">>>="),
[]byte("+"),
[]byte("+="),
[]byte("++"),
[]byte("-"),
[]byte("-="),
[]byte("--"),
[]byte("*"),
[]byte("*="),
[]byte("**"),
[]byte("**="),
[]byte("/"),
[]byte("/="),
[]byte("%"),
[]byte("%="),
[]byte("&"),
[]byte("|"),
[]byte("^"),
[]byte("~"),
[]byte("&="),
[]byte("|="),
[]byte("^="),
[]byte("&&"),
[]byte("||"),
[]byte("??"),
[]byte("&&="),
[]byte("||="),
[]byte("??="),
[]byte("?."),
[]byte("+"),
[]byte("-"),
[]byte("++"),
[]byte("--"),
[]byte("++"),
[]byte("--"),
}
var reservedWordBytes = [][]byte{
[]byte("Reserved"),
[]byte("await"),
[]byte("break"),
[]byte("case"),
[]byte("catch"),
[]byte("class"),
[]byte("const"),
[]byte("continue"),
[]byte("debugger"),
[]byte("default"),
[]byte("delete"),
[]byte("do"),
[]byte("else"),
[]byte("enum"),
[]byte("export"),
[]byte("extends"),
[]byte("false"),
[]byte("finally"),
[]byte("for"),
[]byte("function"),
[]byte("if"),
[]byte("import"),
[]byte("in"),
[]byte("instanceof"),
[]byte("new"),
[]byte("null"),
[]byte("return"),
[]byte("super"),
[]byte("switch"),
[]byte("this"),
[]byte("throw"),
[]byte("true"),
[]byte("try"),
[]byte("typeof"),
[]byte("yield"),
[]byte("var"),
[]byte("void"),
[]byte("while"),
[]byte("with"),
}
var identifierBytes = [][]byte{
[]byte("Identifier"),
[]byte("as"),
[]byte("async"),
[]byte("from"),
[]byte("get"),
[]byte("implements"),
[]byte("interface"),
[]byte("let"),
[]byte("meta"),
[]byte("of"),
[]byte("package"),
[]byte("private"),
[]byte("protected"),
[]byte("public"),
[]byte("set"),
[]byte("static"),
[]byte("target"),
}
// Bytes returns the string representation of a TokenType.
func (tt TokenType) Bytes() []byte {
if IsOperator(tt) && int(tt-OperatorToken) < len(operatorBytes) {
return operatorBytes[tt-OperatorToken]
} else if IsReservedWord(tt) && int(tt-ReservedToken) < len(reservedWordBytes) {
return reservedWordBytes[tt-ReservedToken]
} else if IsIdentifier(tt) && int(tt-IdentifierToken) < len(identifierBytes) {
return identifierBytes[tt-IdentifierToken]
}
switch tt {
case ErrorToken:
return []byte("Error")
case WhitespaceToken:
return []byte("Whitespace")
case LineTerminatorToken:
return []byte("LineTerminator")
case CommentToken:
return []byte("Comment")
case CommentLineTerminatorToken:
return []byte("CommentLineTerminator")
case StringToken:
return []byte("String")
case TemplateToken:
return []byte("Template")
case TemplateStartToken:
return []byte("TemplateStart")
case TemplateMiddleToken:
return []byte("TemplateMiddle")
case TemplateEndToken:
return []byte("TemplateEnd")
case RegExpToken:
return []byte("RegExp")
case PrivateIdentifierToken:
return []byte("PrivateIdentifier")
case NumericToken:
return []byte("Numeric")
case DecimalToken:
return []byte("Decimal")
case BinaryToken:
return []byte("Binary")
case OctalToken:
return []byte("Octal")
case HexadecimalToken:
return []byte("Hexadecimal")
case BigIntToken:
return []byte("BigInt")
case PunctuatorToken:
return []byte("Punctuator")
case OpenBraceToken:
return []byte("{")
case CloseBraceToken:
return []byte("}")
case OpenParenToken:
return []byte("(")
case CloseParenToken:
return []byte(")")
case OpenBracketToken:
return []byte("[")
case CloseBracketToken:
return []byte("]")
case DotToken:
return []byte(".")
case SemicolonToken:
return []byte(";")
case CommaToken:
return []byte(",")
case QuestionToken:
return []byte("?")
case ColonToken:
return []byte(":")
case ArrowToken:
return []byte("=>")
case EllipsisToken:
return []byte("...")
}
return nil
}
parse-2.5.21/js/util.go 0000664 0000000 0000000 00000001365 14117006534 0014641 0 ustar 00root root 0000000 0000000 package js
// AsIdentifierName returns true if a valid identifier name is given.
func AsIdentifierName(b []byte) bool {
if len(b) == 0 || !identifierStartTable[b[0]] {
return false
}
i := 1
for i < len(b) {
if identifierTable[b[i]] {
i++
} else {
return false
}
}
return true
}
// AsDecimalLiteral returns true if a valid decimal literal is given.
func AsDecimalLiteral(b []byte) bool {
if len(b) == 0 || (b[0] < '0' || '9' < b[0]) && (b[0] != '.' || len(b) == 1) {
return false
} else if b[0] == '0' {
return len(b) == 1
}
i := 1
for i < len(b) && '0' <= b[i] && b[i] <= '9' {
i++
}
if i < len(b) && b[i] == '.' && b[0] != '.' {
i++
for i < len(b) && '0' <= b[i] && b[i] <= '9' {
i++
}
}
return i == len(b)
}
parse-2.5.21/js/util_test.go 0000664 0000000 0000000 00000001365 14117006534 0015700 0 ustar 00root root 0000000 0000000 package js
import (
"testing"
"github.com/tdewolff/test"
)
func TestAsIdentifierName(t *testing.T) {
test.That(t, !AsIdentifierName([]byte("")))
test.That(t, !AsIdentifierName([]byte("5")))
test.That(t, AsIdentifierName([]byte("ab")))
test.That(t, !AsIdentifierName([]byte("a=")))
}
func TestAsDecimalLiteral(t *testing.T) {
test.That(t, !AsDecimalLiteral([]byte("")))
test.That(t, !AsDecimalLiteral([]byte("a")))
test.That(t, AsDecimalLiteral([]byte("12")))
test.That(t, AsDecimalLiteral([]byte("12.56")))
test.That(t, AsDecimalLiteral([]byte(".56")))
test.That(t, !AsDecimalLiteral([]byte(".56a")))
test.That(t, !AsDecimalLiteral([]byte(".")))
test.That(t, AsDecimalLiteral([]byte("0")))
test.That(t, !AsDecimalLiteral([]byte("00")))
}
parse-2.5.21/js/walk.go 0000664 0000000 0000000 00000011257 14117006534 0014623 0 ustar 00root root 0000000 0000000 package js
// IVisitor represents the AST Visitor
// Each INode encountered by `Walk` is passed to `Enter`, children nodes will be ignored if the returned IVisitor is nil
// `Exit` is called upon the exit of a node
type IVisitor interface {
Enter(n INode) IVisitor
Exit(n INode)
}
// Walk traverses an AST in depth-first order
func Walk(v IVisitor, n INode) {
if n == nil {
return
}
if v = v.Enter(n); v == nil {
return
}
defer v.Exit(n)
switch n := n.(type) {
case *AST:
Walk(v, &n.BlockStmt)
case *Var:
return
case *BlockStmt:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, n.List[i])
}
}
case *EmptyStmt:
return
case *ExprStmt:
Walk(v, n.Value)
case *IfStmt:
Walk(v, n.Body)
Walk(v, n.Else)
Walk(v, n.Cond)
case *DoWhileStmt:
Walk(v, n.Body)
Walk(v, n.Cond)
case *WhileStmt:
Walk(v, n.Body)
Walk(v, n.Cond)
case *ForStmt:
if n.Body != nil {
Walk(v, n.Body)
}
Walk(v, n.Init)
Walk(v, n.Cond)
Walk(v, n.Post)
case *ForInStmt:
if n.Body != nil {
Walk(v, n.Body)
}
Walk(v, n.Init)
Walk(v, n.Value)
case *ForOfStmt:
if n.Body != nil {
Walk(v, n.Body)
}
Walk(v, n.Init)
Walk(v, n.Value)
case *CaseClause:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, n.List[i])
}
}
Walk(v, n.Cond)
case *SwitchStmt:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
Walk(v, n.Init)
case *BranchStmt:
return
case *ReturnStmt:
Walk(v, n.Value)
case *WithStmt:
Walk(v, n.Body)
Walk(v, n.Cond)
case *LabelledStmt:
Walk(v, n.Value)
case *ThrowStmt:
Walk(v, n.Value)
case *TryStmt:
if n.Body != nil {
Walk(v, n.Body)
}
if n.Catch != nil {
Walk(v, n.Catch)
}
if n.Finally != nil {
Walk(v, n.Finally)
}
Walk(v, n.Binding)
case *DebuggerStmt:
return
case *Alias:
return
case *ImportStmt:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
case *ExportStmt:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
Walk(v, n.Decl)
case *DirectivePrologueStmt:
return
case *PropertyName:
Walk(v, &n.Literal)
Walk(v, n.Computed)
case *BindingArray:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
Walk(v, n.Rest)
case *BindingObjectItem:
if n.Key != nil {
Walk(v, n.Key)
}
Walk(v, &n.Value)
case *BindingObject:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
if n.Rest != nil {
Walk(v, n.Rest)
}
case *BindingElement:
Walk(v, n.Binding)
Walk(v, n.Default)
case *VarDecl:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
case *Params:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
Walk(v, n.Rest)
case *FuncDecl:
Walk(v, &n.Body)
Walk(v, &n.Params)
if n.Name != nil {
Walk(v, n.Name)
}
case *MethodDecl:
Walk(v, &n.Body)
Walk(v, &n.Params)
Walk(v, &n.Name)
case *FieldDefinition:
Walk(v, &n.Name)
Walk(v, n.Init)
case *ClassDecl:
if n.Name != nil {
Walk(v, n.Name)
}
Walk(v, n.Extends)
if n.Definitions != nil {
for i := 0; i < len(n.Definitions); i++ {
Walk(v, &n.Definitions[i])
}
}
if n.Methods != nil {
for i := 0; i < len(n.Definitions); i++ {
Walk(v, &n.Definitions[i])
}
}
case *LiteralExpr:
return
case *Element:
Walk(v, n.Value)
case *ArrayExpr:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
case *Property:
if n.Name != nil {
Walk(v, n.Name)
}
Walk(v, n.Value)
Walk(v, n.Init)
case *ObjectExpr:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
case *TemplatePart:
Walk(v, n.Expr)
case *TemplateExpr:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
Walk(v, n.Tag)
case *GroupExpr:
Walk(v, n.X)
case *IndexExpr:
Walk(v, n.X)
Walk(v, n.Y)
case *DotExpr:
Walk(v, n.X)
Walk(v, &n.Y)
case *NewTargetExpr:
return
case *ImportMetaExpr:
return
case *Arg:
Walk(v, n.Value)
case *Args:
if n.List != nil {
for i := 0; i < len(n.List); i++ {
Walk(v, &n.List[i])
}
}
case *NewExpr:
if n.Args != nil {
Walk(v, n.Args)
}
Walk(v, n.X)
case *CallExpr:
Walk(v, &n.Args)
Walk(v, n.X)
case *OptChainExpr:
Walk(v, n.X)
Walk(v, n.Y)
case *UnaryExpr:
Walk(v, n.X)
case *BinaryExpr:
Walk(v, n.X)
Walk(v, n.Y)
case *CondExpr:
Walk(v, n.Cond)
Walk(v, n.X)
Walk(v, n.Y)
case *YieldExpr:
Walk(v, n.X)
case *ArrowFunc:
Walk(v, &n.Body)
Walk(v, &n.Params)
default:
return
}
}
parse-2.5.21/js/walk_test.go 0000664 0000000 0000000 00000003353 14117006534 0015660 0 ustar 00root root 0000000 0000000 package js
import (
"bytes"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type walker struct{}
func (w *walker) Enter(n INode) IVisitor {
switch n := n.(type) {
case *Var:
if bytes.Equal(n.Data, []byte("x")) {
n.Data = []byte("obj")
}
}
return w
}
func (w *walker) Exit(n INode) {}
func TestWalk(t *testing.T) {
js := `
if (true) {
for (i = 0; i < 1; i++) {
x.y = i
}
}`
ast, err := Parse(parse.NewInputString(js))
if err != nil {
t.Fatal(err)
}
Walk(&walker{}, ast)
t.Run("TestWalk", func(t *testing.T) {
test.String(t, ast.JS(), "if (true) { for (i = 0; i < 1; i++) { obj.y = i; }; }; ")
})
}
func TestWalkNilNode(t *testing.T) {
nodes := []INode{
&AST{},
&Var{},
&BlockStmt{},
&EmptyStmt{},
&ExprStmt{},
&IfStmt{},
&DoWhileStmt{},
&WhileStmt{},
&ForStmt{},
&ForInStmt{},
&ForOfStmt{},
&CaseClause{},
&SwitchStmt{},
&BranchStmt{},
&ReturnStmt{},
&WithStmt{},
&LabelledStmt{},
&ThrowStmt{},
&TryStmt{},
&DebuggerStmt{},
&Alias{},
&ImportStmt{},
&ExportStmt{},
&DirectivePrologueStmt{},
&PropertyName{},
&BindingArray{},
&BindingObjectItem{},
&BindingObject{},
&BindingElement{},
&VarDecl{},
&Params{},
&FuncDecl{},
&MethodDecl{},
&FieldDefinition{},
&ClassDecl{},
&LiteralExpr{},
&Element{},
&ArrayExpr{},
&Property{},
&ObjectExpr{},
&TemplatePart{},
&TemplateExpr{},
&GroupExpr{},
&IndexExpr{},
&DotExpr{},
&NewTargetExpr{},
&ImportMetaExpr{},
&Arg{},
&Args{},
&NewExpr{},
&CallExpr{},
&OptChainExpr{},
&UnaryExpr{},
&BinaryExpr{},
&CondExpr{},
&YieldExpr{},
&ArrowFunc{},
}
t.Run("TestWalkNilNode", func(t *testing.T) {
for _, n := range nodes {
Walk(&walker{}, n)
}
})
}
parse-2.5.21/json/ 0000775 0000000 0000000 00000000000 14117006534 0013665 5 ustar 00root root 0000000 0000000 parse-2.5.21/json/README.md 0000664 0000000 0000000 00000003421 14117006534 0015144 0 ustar 00root root 0000000 0000000 # JSON [](https://pkg.go.dev/github.com/tdewolff/parse/v2/json?tab=doc)
This package is a JSON lexer (ECMA-404) written in [Go][1]. It follows the specification at [JSON](http://json.org/). The lexer takes an io.Reader and converts it into tokens until the EOF.
## Installation
Run the following command
go get -u github.com/tdewolff/parse/v2/json
or add the following import and run project with `go get`
import "github.com/tdewolff/parse/v2/json"
## Parser
### Usage
The following initializes a new Parser with io.Reader `r`:
``` go
p := json.NewParser(parse.NewInput(r))
```
To tokenize until EOF an error, use:
``` go
for {
gt, text := p.Next()
switch gt {
case json.ErrorGrammar:
// error or EOF set in p.Err()
return
// ...
}
}
```
All grammars:
``` go
ErrorGrammar GrammarType = iota // extra grammar when errors occur
WhitespaceGrammar // space \t \r \n
LiteralGrammar // null true false
NumberGrammar
StringGrammar
StartObjectGrammar // {
EndObjectGrammar // }
StartArrayGrammar // [
EndArrayGrammar // ]
```
### Examples
``` go
package main
import (
"os"
"github.com/tdewolff/parse/v2/json"
)
// Tokenize JSON from stdin.
func main() {
p := json.NewParser(parse.NewInput(os.Stdin))
for {
gt, text := p.Next()
switch gt {
case json.ErrorGrammar:
if p.Err() != io.EOF {
fmt.Println("Error on line", p.Line(), ":", p.Err())
}
return
case json.LiteralGrammar:
fmt.Println("Literal", string(text))
case json.NumberGrammar:
fmt.Println("Number", string(text))
// ...
}
}
}
```
## License
Released under the [MIT license](https://github.com/tdewolff/parse/blob/master/LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/json/parse.go 0000664 0000000 0000000 00000016320 14117006534 0015330 0 ustar 00root root 0000000 0000000 // Package json is a JSON parser following the specifications at http://json.org/.
package json
import (
"strconv"
"github.com/tdewolff/parse/v2"
)
// GrammarType determines the type of grammar
type GrammarType uint32
// GrammarType values.
const (
ErrorGrammar GrammarType = iota // extra grammar when errors occur
WhitespaceGrammar
LiteralGrammar
NumberGrammar
StringGrammar
StartObjectGrammar // {
EndObjectGrammar // }
StartArrayGrammar // [
EndArrayGrammar // ]
)
// String returns the string representation of a GrammarType.
func (gt GrammarType) String() string {
switch gt {
case ErrorGrammar:
return "Error"
case WhitespaceGrammar:
return "Whitespace"
case LiteralGrammar:
return "Literal"
case NumberGrammar:
return "Number"
case StringGrammar:
return "String"
case StartObjectGrammar:
return "StartObject"
case EndObjectGrammar:
return "EndObject"
case StartArrayGrammar:
return "StartArray"
case EndArrayGrammar:
return "EndArray"
}
return "Invalid(" + strconv.Itoa(int(gt)) + ")"
}
////////////////////////////////////////////////////////////////
// State determines the current state the parser is in.
type State uint32
// State values.
const (
ValueState State = iota // extra token when errors occur
ObjectKeyState
ObjectValueState
ArrayState
)
// String returns the string representation of a State.
func (state State) String() string {
switch state {
case ValueState:
return "Value"
case ObjectKeyState:
return "ObjectKey"
case ObjectValueState:
return "ObjectValue"
case ArrayState:
return "Array"
}
return "Invalid(" + strconv.Itoa(int(state)) + ")"
}
////////////////////////////////////////////////////////////////
// Parser is the state for the lexer.
type Parser struct {
r *parse.Input
state []State
err error
needComma bool
}
// NewParser returns a new Parser for a given io.Reader.
func NewParser(r *parse.Input) *Parser {
return &Parser{
r: r,
state: []State{ValueState},
}
}
// Err returns the error encountered during tokenization, this is often io.EOF but also other errors can be returned.
func (p *Parser) Err() error {
if p.err != nil {
return p.err
}
return p.r.Err()
}
// State returns the state the parser is currently in (ie. which token is expected).
func (p *Parser) State() State {
return p.state[len(p.state)-1]
}
// Next returns the next Grammar. It returns ErrorGrammar when an error was encountered. Using Err() one can retrieve the error message.
func (p *Parser) Next() (GrammarType, []byte) {
p.moveWhitespace()
c := p.r.Peek(0)
state := p.state[len(p.state)-1]
if c == ',' {
if state != ArrayState && state != ObjectKeyState {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: unexpected comma character")
return ErrorGrammar, nil
}
p.r.Move(1)
p.moveWhitespace()
p.needComma = false
c = p.r.Peek(0)
}
p.r.Skip()
if p.needComma && c != '}' && c != ']' && c != 0 {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: expected comma character or an array or object ending")
return ErrorGrammar, nil
} else if c == '{' {
p.state = append(p.state, ObjectKeyState)
p.r.Move(1)
return StartObjectGrammar, p.r.Shift()
} else if c == '}' {
if state != ObjectKeyState {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: unexpected right brace character")
return ErrorGrammar, nil
}
p.needComma = true
p.state = p.state[:len(p.state)-1]
if p.state[len(p.state)-1] == ObjectValueState {
p.state[len(p.state)-1] = ObjectKeyState
}
p.r.Move(1)
return EndObjectGrammar, p.r.Shift()
} else if c == '[' {
p.state = append(p.state, ArrayState)
p.r.Move(1)
return StartArrayGrammar, p.r.Shift()
} else if c == ']' {
p.needComma = true
if state != ArrayState {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: unexpected right bracket character")
return ErrorGrammar, nil
}
p.state = p.state[:len(p.state)-1]
if p.state[len(p.state)-1] == ObjectValueState {
p.state[len(p.state)-1] = ObjectKeyState
}
p.r.Move(1)
return EndArrayGrammar, p.r.Shift()
} else if state == ObjectKeyState {
if c != '"' || !p.consumeStringToken() {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: expected object key to be a quoted string")
return ErrorGrammar, nil
}
n := p.r.Pos()
p.moveWhitespace()
if c := p.r.Peek(0); c != ':' {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: expected colon character after object key")
return ErrorGrammar, nil
}
p.r.Move(1)
p.state[len(p.state)-1] = ObjectValueState
return StringGrammar, p.r.Shift()[:n]
} else {
p.needComma = true
if state == ObjectValueState {
p.state[len(p.state)-1] = ObjectKeyState
}
if c == '"' && p.consumeStringToken() {
return StringGrammar, p.r.Shift()
} else if p.consumeNumberToken() {
return NumberGrammar, p.r.Shift()
} else if p.consumeLiteralToken() {
return LiteralGrammar, p.r.Shift()
}
c := p.r.Peek(0) // pick up movement from consumeStringToken to detect NULL or EOF
if c == 0 && p.r.Err() == nil {
p.err = parse.NewErrorLexer(p.r, "JSON parse error: unexpected NULL character")
return ErrorGrammar, nil
} else if c == 0 { // EOF
return ErrorGrammar, nil
}
}
p.err = parse.NewErrorLexer(p.r, "JSON parse error: unexpected character '%c'", c)
return ErrorGrammar, nil
}
////////////////////////////////////////////////////////////////
/*
The following functions follow the specifications at http://json.org/
*/
func (p *Parser) moveWhitespace() {
for {
if c := p.r.Peek(0); c != ' ' && c != '\n' && c != '\r' && c != '\t' {
break
}
p.r.Move(1)
}
}
func (p *Parser) consumeLiteralToken() bool {
c := p.r.Peek(0)
if c == 't' && p.r.Peek(1) == 'r' && p.r.Peek(2) == 'u' && p.r.Peek(3) == 'e' {
p.r.Move(4)
return true
} else if c == 'f' && p.r.Peek(1) == 'a' && p.r.Peek(2) == 'l' && p.r.Peek(3) == 's' && p.r.Peek(4) == 'e' {
p.r.Move(5)
return true
} else if c == 'n' && p.r.Peek(1) == 'u' && p.r.Peek(2) == 'l' && p.r.Peek(3) == 'l' {
p.r.Move(4)
return true
}
return false
}
func (p *Parser) consumeNumberToken() bool {
mark := p.r.Pos()
if p.r.Peek(0) == '-' {
p.r.Move(1)
}
c := p.r.Peek(0)
if c >= '1' && c <= '9' {
p.r.Move(1)
for {
if c := p.r.Peek(0); c < '0' || c > '9' {
break
}
p.r.Move(1)
}
} else if c != '0' {
p.r.Rewind(mark)
return false
} else {
p.r.Move(1) // 0
}
if c := p.r.Peek(0); c == '.' {
p.r.Move(1)
if c := p.r.Peek(0); c < '0' || c > '9' {
p.r.Move(-1)
return true
}
for {
if c := p.r.Peek(0); c < '0' || c > '9' {
break
}
p.r.Move(1)
}
}
mark = p.r.Pos()
if c := p.r.Peek(0); c == 'e' || c == 'E' {
p.r.Move(1)
if c := p.r.Peek(0); c == '+' || c == '-' {
p.r.Move(1)
}
if c := p.r.Peek(0); c < '0' || c > '9' {
p.r.Rewind(mark)
return true
}
for {
if c := p.r.Peek(0); c < '0' || c > '9' {
break
}
p.r.Move(1)
}
}
return true
}
func (p *Parser) consumeStringToken() bool {
// assume to be on "
p.r.Move(1)
for {
c := p.r.Peek(0)
if c == '"' {
escaped := false
for i := p.r.Pos() - 1; i >= 0; i-- {
if p.r.Lexeme()[i] == '\\' {
escaped = !escaped
} else {
break
}
}
if !escaped {
p.r.Move(1)
break
}
} else if c == 0 {
return false
}
p.r.Move(1)
}
return true
}
parse-2.5.21/json/parse_test.go 0000664 0000000 0000000 00000012137 14117006534 0016371 0 ustar 00root root 0000000 0000000 package json
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type GTs []GrammarType
func TestGrammars(t *testing.T) {
var grammarTests = []struct {
json string
expected []GrammarType
}{
{" \t\n\r", GTs{}}, // WhitespaceGrammar
{"null", GTs{LiteralGrammar}},
{"[]", GTs{StartArrayGrammar, EndArrayGrammar}},
{"15.2", GTs{NumberGrammar}},
{"0.4", GTs{NumberGrammar}},
{"5e9", GTs{NumberGrammar}},
{"-4E-3", GTs{NumberGrammar}},
{"true", GTs{LiteralGrammar}},
{"false", GTs{LiteralGrammar}},
{"null", GTs{LiteralGrammar}},
{`""`, GTs{StringGrammar}},
{`"abc"`, GTs{StringGrammar}},
{`"\""`, GTs{StringGrammar}},
{`"\\"`, GTs{StringGrammar}},
{"{}", GTs{StartObjectGrammar, EndObjectGrammar}},
{`{"a": "b", "c": "d"}`, GTs{StartObjectGrammar, StringGrammar, StringGrammar, StringGrammar, StringGrammar, EndObjectGrammar}},
{`{"a": [1, 2], "b": {"c": 3}}`, GTs{StartObjectGrammar, StringGrammar, StartArrayGrammar, NumberGrammar, NumberGrammar, EndArrayGrammar, StringGrammar, StartObjectGrammar, StringGrammar, NumberGrammar, EndObjectGrammar, EndObjectGrammar}},
{"[null,]", GTs{StartArrayGrammar, LiteralGrammar, EndArrayGrammar}},
}
for _, tt := range grammarTests {
t.Run(tt.json, func(t *testing.T) {
p := NewParser(parse.NewInputString(tt.json))
i := 0
for {
grammar, _ := p.Next()
if grammar == ErrorGrammar {
test.T(t, p.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
} else if grammar == WhitespaceGrammar {
continue
}
test.That(t, i < len(tt.expected), "index", i, "must not exceed expected grammar types size", len(tt.expected))
if i < len(tt.expected) {
test.T(t, grammar, tt.expected[i], "grammar types must match")
}
i++
}
})
}
// coverage
for i := 0; ; i++ {
if GrammarType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
for i := 0; ; i++ {
if State(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
}
func TestGrammarsErrorEOF(t *testing.T) {
var grammarErrorTests = []struct {
json string
}{
{`{"":"`},
{"\"a\\"},
}
for _, tt := range grammarErrorTests {
t.Run(tt.json, func(t *testing.T) {
p := NewParser(parse.NewInputString(tt.json))
for {
grammar, _ := p.Next()
if grammar == ErrorGrammar {
test.T(t, p.Err(), io.EOF)
break
}
}
})
}
}
func TestGrammarsError(t *testing.T) {
var grammarErrorTests = []struct {
json string
col int
}{
{"true, false", 5},
{"[true false]", 7},
{"]", 1},
{"}", 1},
{"{0: 1}", 2},
{"{\"a\" 1}", 6},
{"1.", 2},
{"1e+", 2},
{"[true, \x00]", 8},
{"\"string\x00\"", 8},
{"{\"id\": noquote}", 8},
{"{\"id\"\x00: 5}", 6},
{"{\"id: \x00}", 7},
{"{\"id: 5\x00", 8},
}
for _, tt := range grammarErrorTests {
t.Run(tt.json, func(t *testing.T) {
p := NewParser(parse.NewInputString(tt.json))
for {
grammar, _ := p.Next()
if grammar == ErrorGrammar {
if perr, ok := p.Err().(*parse.Error); ok {
_, col, _ := perr.Position()
test.T(t, col, tt.col)
} else {
test.Fail(t, "not a parse error:", p.Err())
}
break
}
}
})
}
}
func TestStates(t *testing.T) {
var stateTests = []struct {
json string
expected []State
}{
{"null", []State{ValueState}},
{"[null]", []State{ArrayState, ArrayState, ValueState}},
{"{\"\":null}", []State{ObjectKeyState, ObjectValueState, ObjectKeyState, ValueState}},
}
for _, tt := range stateTests {
t.Run(tt.json, func(t *testing.T) {
p := NewParser(parse.NewInputString(tt.json))
i := 0
for {
grammar, _ := p.Next()
state := p.State()
if grammar == ErrorGrammar {
test.T(t, p.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
} else if grammar == WhitespaceGrammar {
continue
}
test.That(t, i < len(tt.expected), "index", i, "must not exceed expected states size", len(tt.expected))
if i < len(tt.expected) {
test.T(t, state, tt.expected[i], "states must match")
}
i++
}
})
}
}
func TestOffset(t *testing.T) {
z := parse.NewInputString(`{"key": [5, "string", null, true]}`)
p := NewParser(z)
test.T(t, z.Offset(), 0)
_, _ = p.Next()
test.T(t, z.Offset(), 1) // {
_, _ = p.Next()
test.T(t, z.Offset(), 7) // "key":
_, _ = p.Next()
test.T(t, z.Offset(), 9) // [
_, _ = p.Next()
test.T(t, z.Offset(), 10) // 5
_, _ = p.Next()
test.T(t, z.Offset(), 20) // , "string"
_, _ = p.Next()
test.T(t, z.Offset(), 26) // , null
_, _ = p.Next()
test.T(t, z.Offset(), 32) // , true
_, _ = p.Next()
test.T(t, z.Offset(), 33) // ]
_, _ = p.Next()
test.T(t, z.Offset(), 34) // }
}
////////////////////////////////////////////////////////////////
func ExampleNewParser() {
p := NewParser(parse.NewInputString(`{"key": 5}`))
out := ""
for {
state := p.State()
gt, data := p.Next()
if gt == ErrorGrammar {
break
}
out += string(data)
if state == ObjectKeyState && gt != EndObjectGrammar {
out += ":"
}
// not handling comma insertion
}
fmt.Println(out)
// Output: {"key":5}
}
parse-2.5.21/position.go 0000664 0000000 0000000 00000004025 14117006534 0015110 0 ustar 00root root 0000000 0000000 package parse
import (
"fmt"
"io"
"strings"
"unicode"
)
// Position returns the line and column number for a certain position in a file. It is useful for recovering the position in a file that caused an error.
// It only treates \n, \r, and \r\n as newlines, which might be different from some languages also recognizing \f, \u2028, and \u2029 to be newlines.
func Position(r io.Reader, offset int) (line, col int, context string) {
l := NewInput(r)
line = 1
for l.Pos() < offset {
c := l.Peek(0)
n := 1
newline := false
if c == '\n' {
newline = true
} else if c == '\r' {
if l.Peek(1) == '\n' {
newline = true
n = 2
} else {
newline = true
}
} else if c >= 0xC0 {
var r rune
if r, n = l.PeekRune(0); r == '\u2028' || r == '\u2029' {
newline = true
}
} else if c == 0 && l.Err() != nil {
break
}
if 1 < n && offset < l.Pos()+n {
break
}
l.Move(n)
if newline {
line++
offset -= l.Pos()
l.Skip()
}
}
col = len([]rune(string(l.Lexeme()))) + 1
context = positionContext(l, line, col)
return
}
func positionContext(l *Input, line, col int) (context string) {
for {
c := l.Peek(0)
if c == 0 && l.Err() != nil || c == '\n' || c == '\r' {
break
}
l.Move(1)
}
rs := []rune(string(l.Lexeme()))
// cut off front or rear of context to stay between 60 characters
limit := 60
offset := 20
ellipsisFront := ""
ellipsisRear := ""
if limit < len(rs) {
if col <= limit-offset {
ellipsisRear = "..."
rs = rs[:limit-3]
} else if col >= len(rs)-offset-3 {
ellipsisFront = "..."
col -= len(rs) - offset - offset - 7
rs = rs[len(rs)-offset-offset-4:]
} else {
ellipsisFront = "..."
ellipsisRear = "..."
rs = rs[col-offset-1 : col+offset]
col = offset + 4
}
}
// replace unprintable characters by a space
for i, r := range rs {
if !unicode.IsGraphic(r) {
rs[i] = '·'
}
}
context += fmt.Sprintf("%5d: %s%s%s\n", line, ellipsisFront, string(rs), ellipsisRear)
context += fmt.Sprintf("%s^", strings.Repeat(" ", 6+col))
return
}
parse-2.5.21/position_test.go 0000664 0000000 0000000 00000005415 14117006534 0016153 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"fmt"
"strings"
"testing"
"github.com/tdewolff/test"
)
func TestPosition(t *testing.T) {
var newlineTests = []struct {
offset int
buf string
line int
col int
}{
// \u2028, \u2029, and \u2318 are three bytes long
{0, "x", 1, 1},
{1, "xx", 1, 2},
{2, "x\nx", 2, 1},
{2, "\n\nx", 3, 1},
{3, "\nxxx", 2, 3},
{2, "\r\nx", 2, 1},
{1, "\rx", 2, 1},
{3, "\u2028x", 2, 1},
{3, "\u2029x", 2, 1},
// edge cases
{0, "", 1, 1},
{2, "x", 1, 2},
{0, "\nx", 1, 1},
{1, "\r\ny", 1, 1},
{-1, "x", 1, 1},
{0, "\x00a", 1, 1},
{2, "a\x00\n", 1, 3},
// unicode
{1, "x\u2028x", 1, 2},
{2, "x\u2028x", 1, 2},
{3, "x\u2028x", 1, 2},
{0, "x\u2318x", 1, 1},
{1, "x\u2318x", 1, 2},
{2, "x\u2318x", 1, 2},
{3, "x\u2318x", 1, 2},
{4, "x\u2318x", 1, 3},
}
for _, tt := range newlineTests {
t.Run(fmt.Sprint(tt.buf, " ", tt.offset), func(t *testing.T) {
r := bytes.NewBufferString(tt.buf)
line, col, _ := Position(r, tt.offset)
test.T(t, line, tt.line, "line")
test.T(t, col, tt.col, "column")
})
}
}
func TestPositionContext(t *testing.T) {
var newlineTests = []struct {
offset int
buf string
context string
}{
{10, "0123456789@123456789012345678901234567890123456789012345678901234567890123456789", "0123456789@1234567890123456789012345678901234567890123456..."}, // 80 characters -> 60 characters
{40, "0123456789012345678901234567890123456789@123456789012345678901234567890123456789", "...01234567890123456789@12345678901234567890..."},
{60, "012345678901234567890123456789012345678901234567890123456789@12345678901234567890", "...78901234567890123456789@12345678901234567890"},
{60, "012345678901234567890123456789012345678901234567890123456789@12345678901234567890123", "...01234567890123456789@12345678901234567890123"},
{60, "012345678901234567890123456789012345678901234567890123456789@123456789012345678901234", "...01234567890123456789@12345678901234567890..."},
{60, "0123456789012345678901234567890123456789ÎÎÎÎÎÎÎÎÎÎ@123456789012345678901234567890", "...0123456789ÎÎÎÎÎÎÎÎÎÎ@12345678901234567890..."},
{60, "012345678901234567890123456789012345678912 45678 0123456789@12345678901234567890", "...789·12·45678·0123456789@12345678901234567890"},
}
for _, tt := range newlineTests {
t.Run(fmt.Sprint(tt.buf, " ", tt.offset), func(t *testing.T) {
r := bytes.NewBufferString(tt.buf)
_, _, context := Position(r, tt.offset)
i := strings.IndexByte(context, '\n')
pointer := context[i+1:]
context = context[:i]
test.T(t, context[7:], tt.context)
// check if @ and ^ are at the same position
j := strings.IndexByte(context, '@')
k := strings.IndexByte(pointer, '^')
test.T(t, len([]rune(pointer[:k])), len([]rune(context[:j])))
})
}
}
parse-2.5.21/strconv/ 0000775 0000000 0000000 00000000000 14117006534 0014412 5 ustar 00root root 0000000 0000000 parse-2.5.21/strconv/float.go 0000664 0000000 0000000 00000013101 14117006534 0016042 0 ustar 00root root 0000000 0000000 package strconv
import (
"math"
)
var float64pow10 = []float64{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18, 1e19,
1e20, 1e21, 1e22,
}
// ParseFloat parses a byte-slice and returns the float it represents.
// If an invalid character is encountered, it will stop there.
func ParseFloat(b []byte) (float64, int) {
i := 0
neg := false
if i < len(b) && (b[i] == '+' || b[i] == '-') {
neg = b[i] == '-'
i++
}
start := i
dot := -1
trunk := -1
n := uint64(0)
for ; i < len(b); i++ {
c := b[i]
if c >= '0' && c <= '9' {
if trunk == -1 {
if n > math.MaxUint64/10 {
trunk = i
} else {
n *= 10
n += uint64(c - '0')
}
}
} else if dot == -1 && c == '.' {
dot = i
} else {
break
}
}
if i == start || i == start+1 && dot == start {
return 0.0, 0
}
f := float64(n)
if neg {
f = -f
}
mantExp := int64(0)
if dot != -1 {
if trunk == -1 {
trunk = i
}
mantExp = int64(trunk - dot - 1)
} else if trunk != -1 {
mantExp = int64(trunk - i)
}
expExp := int64(0)
if i < len(b) && (b[i] == 'e' || b[i] == 'E') {
startExp := i
i++
if e, expLen := ParseInt(b[i:]); expLen > 0 {
expExp = e
i += expLen
} else {
i = startExp
}
}
exp := expExp - mantExp
// copied from strconv/atof.go
if exp == 0 {
return f, i
} else if exp > 0 && exp <= 15+22 { // int * 10^k
// If exponent is big but number of digits is not,
// can move a few zeros into the integer part.
if exp > 22 {
f *= float64pow10[exp-22]
exp = 22
}
if f <= 1e15 && f >= -1e15 {
return f * float64pow10[exp], i
}
} else if exp < 0 && exp >= -22 { // int / 10^k
return f / float64pow10[-exp], i
}
f *= math.Pow10(int(-mantExp))
return f * math.Pow10(int(expExp)), i
}
const log2 = 0.3010299956639812
func float64exp(f float64) int {
exp2 := 0
if f != 0.0 {
x := math.Float64bits(f)
exp2 = int(x>>(64-11-1))&0x7FF - 1023 + 1
}
exp10 := float64(exp2) * log2
if exp10 < 0 {
exp10 -= 1.0
}
return int(exp10)
}
// AppendFloat appends a float to `b` with precision `prec`. It returns the new slice and whether successful or not. Precision is the number of decimals to display, thus prec + 1 == number of significant digits.
func AppendFloat(b []byte, f float64, prec int) ([]byte, bool) {
if math.IsNaN(f) || math.IsInf(f, 0) {
return b, false
}
neg := false
if f < 0.0 {
f = -f
neg = true
}
if prec < 0 || 17 < prec {
prec = 17 // maximum number of significant digits in double
}
prec -= float64exp(f) // number of digits in front of the dot
f *= math.Pow10(prec)
// calculate mantissa and exponent
mant := int64(f)
mantLen := LenInt(mant)
mantExp := mantLen - prec - 1
if mant == 0 {
return append(b, '0'), true
}
// expLen is zero for positive exponents, because positive exponents are determined later on in the big conversion loop
exp := 0
expLen := 0
if mantExp > 0 {
// positive exponent is determined in the loop below
// but if we initially decreased the exponent to fit in an integer, we can't set the new exponent in the loop alone,
// since the number of zeros at the end determines the positive exponent in the loop, and we just artificially lost zeros
if prec < 0 {
exp = mantExp
}
expLen = 1 + LenInt(int64(exp)) // e + digits
} else if mantExp < -3 {
exp = mantExp
expLen = 2 + LenInt(int64(exp)) // e + minus + digits
} else if mantExp < -1 {
mantLen += -mantExp - 1 // extra zero between dot and first digit
}
// reserve space in b
i := len(b)
maxLen := 1 + mantLen + expLen // dot + mantissa digits + exponent
if neg {
maxLen++
}
if i+maxLen > cap(b) {
b = append(b, make([]byte, maxLen)...)
} else {
b = b[:i+maxLen]
}
// write to string representation
if neg {
b[i] = '-'
i++
}
// big conversion loop, start at the end and move to the front
// initially print trailing zeros and remove them later on
// for example if the first non-zero digit is three positions in front of the dot, it will overwrite the zeros with a positive exponent
zero := true
last := i + mantLen // right-most position of digit that is non-zero + dot
dot := last - prec - exp // position of dot
j := last
for mant > 0 {
if j == dot {
b[j] = '.'
j--
}
newMant := mant / 10
digit := mant - 10*newMant
if zero && digit > 0 {
// first non-zero digit, if we are still behind the dot we can trim the end to this position
// otherwise trim to the dot (including the dot)
if j > dot {
i = j + 1
// decrease negative exponent further to get rid of dot
if exp < 0 {
newExp := exp - (j - dot)
// getting rid of the dot shouldn't lower the exponent to more digits (e.g. -9 -> -10)
if LenInt(int64(newExp)) == LenInt(int64(exp)) {
exp = newExp
dot = j
j--
i--
}
}
} else {
i = dot
}
last = j
zero = false
}
b[j] = '0' + byte(digit)
j--
mant = newMant
}
if j > dot {
// extra zeros behind the dot
for j > dot {
b[j] = '0'
j--
}
b[j] = '.'
} else if last+3 < dot {
// add positive exponent because we have 3 or more zeros in front of the dot
i = last + 1
exp = dot - last - 1
} else if j == dot {
// handle 0.1
b[j] = '.'
}
// exponent
if exp != 0 {
if exp == 1 {
b[i] = '0'
i++
} else if exp == 2 {
b[i] = '0'
b[i+1] = '0'
i += 2
} else {
b[i] = 'e'
i++
if exp < 0 {
b[i] = '-'
i++
exp = -exp
}
i += LenInt(int64(exp))
j := i
for exp > 0 {
newExp := exp / 10
digit := exp - 10*newExp
j--
b[j] = '0' + byte(digit)
exp = newExp
}
}
}
return b[:i], true
}
parse-2.5.21/strconv/float_test.go 0000664 0000000 0000000 00000010424 14117006534 0017106 0 ustar 00root root 0000000 0000000 package strconv
import (
"fmt"
"math"
"math/rand"
"strconv"
"testing"
"github.com/tdewolff/test"
)
func TestParseFloat(t *testing.T) {
floatTests := []struct {
f string
expected float64
}{
{"5", 5},
{"5.1", 5.1},
{"-5.1", -5.1},
{"5.1e-2", 5.1e-2},
{"5.1e+2", 5.1e+2},
{"0.0e1", 0.0e1},
{"18446744073709551620", 18446744073709551620.0},
{"1e23", 1e23},
// TODO: hard to test due to float imprecision
// {"1.7976931348623e+308", 1.7976931348623e+308)
// {"4.9406564584124e-308", 4.9406564584124e-308)
}
for _, tt := range floatTests {
t.Run(fmt.Sprint(tt.f), func(t *testing.T) {
f, n := ParseFloat([]byte(tt.f))
test.T(t, n, len(tt.f))
test.T(t, f, tt.expected)
})
}
}
func TestParseFloatError(t *testing.T) {
floatTests := []struct {
f string
n int
expected float64
}{
{"e1", 0, 0},
{".", 0, 0},
{"1e", 1, 1},
{"1e+", 1, 1},
{"1e+1", 4, 10},
}
for _, tt := range floatTests {
t.Run(fmt.Sprint(tt.f), func(t *testing.T) {
f, n := ParseFloat([]byte(tt.f))
test.T(t, n, tt.n)
test.T(t, f, tt.expected)
})
}
}
func TestAppendFloat(t *testing.T) {
floatTests := []struct {
f float64
prec int
expected string
}{
{0, 6, "0"},
{1, 6, "1"},
{9, 6, "9"},
{9.99999, 6, "9.99999"},
{123, 6, "123"},
{0.123456, 6, ".123456"},
{0.066, 6, ".066"},
{0.0066, 6, ".0066"},
{12e2, 6, "1200"},
{12e3, 6, "12e3"},
{0.1, 6, ".1"},
{0.001, 6, ".001"},
{0.0001, 6, "1e-4"},
{-1, 6, "-1"},
{-123, 6, "-123"},
{-123.456, 6, "-123.456"},
{-12e3, 6, "-12e3"},
{-0.1, 6, "-.1"},
{-0.0001, 6, "-1e-4"},
{0.000100009, 10, "100009e-9"},
{0.0001000009, 10, "1.000009e-4"},
{1e18, 0, "1e18"},
//{1e19, 0, "1e19"},
//{1e19, 18, "1e19"},
{1e1, 0, "10"},
{1e2, 1, "100"},
{1e3, 2, "1e3"},
{1e10, -1, "1e10"},
{1e15, -1, "1e15"},
{1e-5, 6, "1e-5"},
{math.NaN(), 0, ""},
{math.Inf(1), 0, ""},
{math.Inf(-1), 0, ""},
{0, 19, "0"},
{0.000923361977200859392, -1, "9.23361977200859392e-4"},
{1234, 2, "1.23e3"},
{12345, 2, "1.23e4"},
{12.345, 2, "12.3"},
{12.345, 3, "12.34"},
}
for _, tt := range floatTests {
t.Run(fmt.Sprint(tt.f), func(t *testing.T) {
f, _ := AppendFloat([]byte{}, tt.f, tt.prec)
test.String(t, string(f), tt.expected)
})
}
b := make([]byte, 0, 22)
AppendFloat(b, 12.34, -1)
test.String(t, string(b[:5]), "12.34", "in buffer")
}
////////////////////////////////////////////////////////////////
func TestAppendFloatRandom(t *testing.T) {
N := int(1e6)
if testing.Short() {
N = 0
}
r := rand.New(rand.NewSource(99))
//prec := 10
for i := 0; i < N; i++ {
f := r.ExpFloat64()
//f = math.Floor(f*float64(prec)) / float64(prec)
b, _ := AppendFloat([]byte{}, f, -1)
f2, _ := strconv.ParseFloat(string(b), 64)
if math.Abs(f-f2) > 1e-6 {
fmt.Println("Bad:", f, "!=", f2, "in", string(b))
}
}
}
func BenchmarkFloatToBytes1(b *testing.B) {
r := []byte{} //make([]byte, 10)
f := 123.456
for i := 0; i < b.N; i++ {
r = strconv.AppendFloat(r[:0], f, 'g', 6, 64)
}
}
func BenchmarkFloatToBytes2(b *testing.B) {
r := make([]byte, 10)
f := 123.456
for i := 0; i < b.N; i++ {
r, _ = AppendFloat(r[:0], f, 6)
}
}
func BenchmarkModf1(b *testing.B) {
f := 123.456
x := 0.0
for i := 0; i < b.N; i++ {
a, b := math.Modf(f)
x += a + b
}
}
func BenchmarkModf2(b *testing.B) {
f := 123.456
x := 0.0
for i := 0; i < b.N; i++ {
a := float64(int64(f))
b := f - a
x += a + b
}
}
func BenchmarkPrintInt1(b *testing.B) {
X := int64(123456789)
n := LenInt(X)
r := make([]byte, n)
for i := 0; i < b.N; i++ {
x := X
j := n
for x > 0 {
j--
r[j] = '0' + byte(x%10)
x /= 10
}
}
}
func BenchmarkPrintInt2(b *testing.B) {
X := int64(123456789)
n := LenInt(X)
r := make([]byte, n)
for i := 0; i < b.N; i++ {
x := X
j := n
for x > 0 {
j--
newX := x / 10
r[j] = '0' + byte(x-10*newX)
x = newX
}
}
}
var int64pow10 = []int64{
1e0, 1e1, 1e2, 1e3, 1e4, 1e5, 1e6, 1e7, 1e8, 1e9,
1e10, 1e11, 1e12, 1e13, 1e14, 1e15, 1e16, 1e17, 1e18,
}
func BenchmarkPrintInt3(b *testing.B) {
X := int64(123456789)
n := LenInt(X)
r := make([]byte, n)
for i := 0; i < b.N; i++ {
x := X
j := 0
for j < n {
pow := int64pow10[n-j-1]
tmp := x / pow
r[j] = '0' + byte(tmp)
j++
x -= tmp * pow
}
}
}
parse-2.5.21/strconv/int.go 0000664 0000000 0000000 00000002722 14117006534 0015536 0 ustar 00root root 0000000 0000000 package strconv
import (
"math"
)
// ParseInt parses a byte-slice and returns the integer it represents.
// If an invalid character is encountered, it will stop there.
func ParseInt(b []byte) (int64, int) {
i := 0
neg := false
if len(b) > 0 && (b[0] == '+' || b[0] == '-') {
neg = b[0] == '-'
i++
}
start := i
n := uint64(0)
for i < len(b) {
c := b[i]
if n > math.MaxUint64/10 {
return 0, 0
} else if c >= '0' && c <= '9' {
n *= 10
n += uint64(c - '0')
} else {
break
}
i++
}
if i == start {
return 0, 0
}
if !neg && n > uint64(math.MaxInt64) || n > uint64(math.MaxInt64)+1 {
return 0, 0
} else if neg {
return -int64(n), i
}
return int64(n), i
}
// LenInt returns the written length of an integer.
func LenInt(i int64) int {
if i < 0 {
if i == -9223372036854775808 {
return 19
}
i = -i
}
switch {
case i < 10:
return 1
case i < 100:
return 2
case i < 1000:
return 3
case i < 10000:
return 4
case i < 100000:
return 5
case i < 1000000:
return 6
case i < 10000000:
return 7
case i < 100000000:
return 8
case i < 1000000000:
return 9
case i < 10000000000:
return 10
case i < 100000000000:
return 11
case i < 1000000000000:
return 12
case i < 10000000000000:
return 13
case i < 100000000000000:
return 14
case i < 1000000000000000:
return 15
case i < 10000000000000000:
return 16
case i < 100000000000000000:
return 17
case i < 1000000000000000000:
return 18
}
return 19
}
parse-2.5.21/strconv/int_test.go 0000664 0000000 0000000 00000004245 14117006534 0016577 0 ustar 00root root 0000000 0000000 package strconv
import (
"fmt"
"math"
"math/rand"
"testing"
"github.com/tdewolff/test"
)
func TestParseInt(t *testing.T) {
intTests := []struct {
i string
expected int64
}{
{"5", 5},
{"99", 99},
{"999", 999},
{"-5", -5},
{"+5", 5},
{"9223372036854775807", 9223372036854775807},
{"-9223372036854775807", -9223372036854775807},
{"-9223372036854775808", -9223372036854775808},
}
for _, tt := range intTests {
t.Run(fmt.Sprint(tt.i), func(t *testing.T) {
i, n := ParseInt([]byte(tt.i))
test.T(t, n, len(tt.i))
test.T(t, i, tt.expected)
})
}
}
func TestParseIntError(t *testing.T) {
intTests := []struct {
i string
n int
expected int64
}{
{"a", 0, 0},
{"+", 0, 0},
{"9223372036854775808", 0, 0},
{"-9223372036854775809", 0, 0},
{"18446744073709551620", 0, 0},
}
for _, tt := range intTests {
t.Run(fmt.Sprint(tt.i), func(t *testing.T) {
i, n := ParseInt([]byte(tt.i))
test.T(t, n, tt.n)
test.T(t, i, tt.expected)
})
}
}
func TestLenInt(t *testing.T) {
lenIntTests := []struct {
i int64
expected int
}{
{0, 1},
{1, 1},
{10, 2},
{99, 2},
{9223372036854775807, 19},
{-9223372036854775808, 19},
// coverage
{100, 3},
{1000, 4},
{10000, 5},
{100000, 6},
{1000000, 7},
{10000000, 8},
{100000000, 9},
{1000000000, 10},
{10000000000, 11},
{100000000000, 12},
{1000000000000, 13},
{10000000000000, 14},
{100000000000000, 15},
{1000000000000000, 16},
{10000000000000000, 17},
{100000000000000000, 18},
{1000000000000000000, 19},
}
for _, tt := range lenIntTests {
t.Run(fmt.Sprint(tt.i), func(t *testing.T) {
test.T(t, LenInt(tt.i), tt.expected)
})
}
}
////////////////////////////////////////////////////////////////
var num []int64
func TestMain(t *testing.T) {
for j := 0; j < 1000; j++ {
num = append(num, rand.Int63n(1000))
}
}
func BenchmarkLenIntLog(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for j := 0; j < 1000; j++ {
n += int(math.Log10(math.Abs(float64(num[j])))) + 1
}
}
}
func BenchmarkLenIntSwitch(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for j := 0; j < 1000; j++ {
n += LenInt(num[j])
}
}
}
parse-2.5.21/strconv/price.go 0000664 0000000 0000000 00000002540 14117006534 0016044 0 ustar 00root root 0000000 0000000 package strconv
// AppendPrice will append an int64 formatted as a price, where the int64 is the price in cents.
// It does not display whether a price is negative or not.
func AppendPrice(b []byte, price int64, dec bool, milSeparator byte, decSeparator byte) []byte {
if price < 0 {
if price == -9223372036854775808 {
x := []byte("92 233 720 368 547 758 08")
x[2] = milSeparator
x[6] = milSeparator
x[10] = milSeparator
x[14] = milSeparator
x[18] = milSeparator
x[22] = decSeparator
return append(b, x...)
}
price = -price
}
// rounding
if !dec {
firstDec := (price / 10) % 10
if firstDec >= 5 {
price += 100
}
}
// calculate size
n := LenInt(price) - 2
if n > 0 {
n += (n - 1) / 3 // mil separator
} else {
n = 1
}
if dec {
n += 2 + 1 // decimals + dec separator
}
// resize byte slice
i := len(b)
if i+n > cap(b) {
b = append(b, make([]byte, n)...)
} else {
b = b[:i+n]
}
// print fractional-part
i += n - 1
if dec {
for j := 0; j < 2; j++ {
c := byte(price%10) + '0'
price /= 10
b[i] = c
i--
}
b[i] = decSeparator
i--
} else {
price /= 100
}
if price == 0 {
b[i] = '0'
return b
}
// print integer-part
j := 0
for price > 0 {
if j == 3 {
b[i] = milSeparator
i--
j = 0
}
c := byte(price%10) + '0'
price /= 10
b[i] = c
i--
j++
}
return b
}
parse-2.5.21/strconv/price_test.go 0000664 0000000 0000000 00000001431 14117006534 0017101 0 ustar 00root root 0000000 0000000 package strconv
import (
"fmt"
"testing"
"github.com/tdewolff/test"
)
func TestAppendPrice(t *testing.T) {
priceTests := []struct {
price int64
dec bool
expected string
}{
{0, false, "0"},
{0, true, "0.00"},
{100, true, "1.00"},
{-100, true, "1.00"},
{100000, false, "1,000"},
{100000, true, "1,000.00"},
{123456789012, true, "1,234,567,890.12"},
{9223372036854775807, true, "92,233,720,368,547,758.07"},
{-9223372036854775808, true, "92,233,720,368,547,758.08"},
{149, false, "1"},
{150, false, "2"},
}
for _, tt := range priceTests {
t.Run(fmt.Sprint(tt.price), func(t *testing.T) {
price := AppendPrice(make([]byte, 0, 4), tt.price, tt.dec, ',', '.')
test.String(t, string(price), tt.expected, "for", tt.price)
})
}
// coverage
}
parse-2.5.21/tests/ 0000775 0000000 0000000 00000000000 14117006534 0014056 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/README.md 0000664 0000000 0000000 00000000777 14117006534 0015350 0 ustar 00root root 0000000 0000000 # Fuzzing parse
Fuzz testing for `parse` using [go-fuzz](https://github.com/dvyukov/go-fuzz). Pull requests to add more corpora or testers are appreciated.
To run the tests, install `go-fuzz`:
```
go get -u github.com/dvyukov/go-fuzz/go-fuzz github.com/dvyukov/go-fuzz/go-fuzz-build
cd $GOPATH/github.com/tdewolff/parse/tests/number
go-fuzz-build
go-fuzz -bin fuzz-fuzz.zip
```
If restarts is not close to `1/10000`, something is probably wrong. If not finding new corpus for a while, restart the fuzzer.
parse-2.5.21/tests/css-token/ 0000775 0000000 0000000 00000000000 14117006534 0015764 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/css-token/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0017277 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/css-token/corpus/1.txt 0000664 0000000 0000000 00000000017 14117006534 0020176 0 ustar 00root root 0000000 0000000 -test128\@\
'"
parse-2.5.21/tests/css-token/main.go 0000664 0000000 0000000 00000000250 14117006534 0017234 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import (
"github.com/tdewolff/parse/v2/css"
)
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
_ = css.IsIdent(data)
return 1
}
parse-2.5.21/tests/data-uri/ 0000775 0000000 0000000 00000000000 14117006534 0015564 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/data-uri/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0017077 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/data-uri/corpus/1.corpus 0000664 0000000 0000000 00000000062 14117006534 0020472 0 ustar 00root root 0000000 0000000 data:text/vnd-example+xyz;foo=bar;base64,R0lGODdh
parse-2.5.21/tests/data-uri/corpus/2.corpus 0000664 0000000 0000000 00000000216 14117006534 0020474 0 ustar 00root root 0000000 0000000 data:image/png;base64,iVBORw0KGgoAAA
ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4
//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU
5ErkJggg==
parse-2.5.21/tests/data-uri/corpus/3.corpus 0000664 0000000 0000000 00000000073 14117006534 0020476 0 ustar 00root root 0000000 0000000 data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678
parse-2.5.21/tests/data-uri/corpus/4.corpus 0000664 0000000 0000000 00000001304 14117006534 0020475 0 ustar 00root root 0000000 0000000 data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kfr6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////////////////////////////////////////////////////////////////wAARCAAYAEADAREAAhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIRAAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYyJLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k=
parse-2.5.21/tests/data-uri/main.go 0000664 0000000 0000000 00000000300 14117006534 0017030 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import "github.com/tdewolff/parse/v2"
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
data = parse.Copy(data)
_, _, _ = parse.DataURI(data)
return 1
}
parse-2.5.21/tests/dimension/ 0000775 0000000 0000000 00000000000 14117006534 0016043 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/dimension/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0017356 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/dimension/corpus/1.corpus 0000664 0000000 0000000 00000000062 14117006534 0020751 0 ustar 00root root 0000000 0000000 data:text/vnd-example+xyz;foo=bar;base64,R0lGODdh
parse-2.5.21/tests/dimension/corpus/2.corpus 0000664 0000000 0000000 00000000216 14117006534 0020753 0 ustar 00root root 0000000 0000000 data:image/png;base64,iVBORw0KGgoAAA
ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4
//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU
5ErkJggg==
parse-2.5.21/tests/dimension/corpus/3.corpus 0000664 0000000 0000000 00000000073 14117006534 0020755 0 ustar 00root root 0000000 0000000 data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678
parse-2.5.21/tests/dimension/corpus/4.corpus 0000664 0000000 0000000 00000001304 14117006534 0020754 0 ustar 00root root 0000000 0000000 data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kfr6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////////////////////////////////////////////////////////////////wAARCAAYAEADAREAAhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIRAAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYyJLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k=
parse-2.5.21/tests/dimension/main.go 0000664 0000000 0000000 00000000246 14117006534 0017320 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import "github.com/tdewolff/parse/v2"
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
_, _ = parse.Dimension(data)
return 1
}
parse-2.5.21/tests/mediatype/ 0000775 0000000 0000000 00000000000 14117006534 0016037 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/mediatype/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0017352 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/mediatype/corpus/1.corpus 0000664 0000000 0000000 00000000062 14117006534 0020745 0 ustar 00root root 0000000 0000000 data:text/vnd-example+xyz;foo=bar;base64,R0lGODdh
parse-2.5.21/tests/mediatype/corpus/2.corpus 0000664 0000000 0000000 00000000216 14117006534 0020747 0 ustar 00root root 0000000 0000000 data:image/png;base64,iVBORw0KGgoAAA
ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4
//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU
5ErkJggg==
parse-2.5.21/tests/mediatype/corpus/3.corpus 0000664 0000000 0000000 00000000073 14117006534 0020751 0 ustar 00root root 0000000 0000000 data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678
parse-2.5.21/tests/mediatype/corpus/4.corpus 0000664 0000000 0000000 00000001304 14117006534 0020750 0 ustar 00root root 0000000 0000000 data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kfr6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////////////////////////////////////////////////////////////////wAARCAAYAEADAREAAhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIRAAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYyJLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k=
parse-2.5.21/tests/mediatype/main.go 0000664 0000000 0000000 00000000246 14117006534 0017314 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import "github.com/tdewolff/parse/v2"
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
_, _ = parse.Mediatype(data)
return 1
}
parse-2.5.21/tests/number/ 0000775 0000000 0000000 00000000000 14117006534 0015346 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/number/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0016661 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/number/corpus/1.corpus 0000664 0000000 0000000 00000000062 14117006534 0020254 0 ustar 00root root 0000000 0000000 data:text/vnd-example+xyz;foo=bar;base64,R0lGODdh
parse-2.5.21/tests/number/corpus/2.corpus 0000664 0000000 0000000 00000000216 14117006534 0020256 0 ustar 00root root 0000000 0000000 data:image/png;base64,iVBORw0KGgoAAA
ANSUhEUgAAAAUAAAAFCAYAAACNbyblAAAAHElEQVQI12P4
//8/w38GIAXDIBKE0DHxgljNBAAO9TXL0Y4OHwAAAABJRU
5ErkJggg==
parse-2.5.21/tests/number/corpus/3.corpus 0000664 0000000 0000000 00000000073 14117006534 0020260 0 ustar 00root root 0000000 0000000 data:text/plain;charset=UTF-8;page=21,the%20data:1234,5678
parse-2.5.21/tests/number/corpus/4.corpus 0000664 0000000 0000000 00000001304 14117006534 0020257 0 ustar 00root root 0000000 0000000 data:image/jpeg;base64,/9j/4AAQSkZJRgABAQEAYABgAAD/2wBDADIiJSwlHzIsKSw4NTI7S31RS0VFS5ltc1p9tZ++u7Kfr6zI4f/zyNT/16yv+v/9////////wfD/////////////2wBDATU4OEtCS5NRUZP/zq/O////////////////////////////////////////////////////////////////////wAARCAAYAEADAREAAhEBAxEB/8QAGQAAAgMBAAAAAAAAAAAAAAAAAQMAAgQF/8QAJRABAAIBBAEEAgMAAAAAAAAAAQIRAAMSITEEEyJBgTORUWFx/8QAFAEBAAAAAAAAAAAAAAAAAAAAAP/EABQRAQAAAAAAAAAAAAAAAAAAAAD/2gAMAwEAAhEDEQA/AOgM52xQDrjvAV5Xv0vfKUALlTQfeBm0HThMNHXkL0Lw/swN5qgA8yT4MCS1OEOJV8mBz9Z05yfW8iSx7p4j+jA1aD6Wj7ZMzstsfvAas4UyRHvjrAkC9KhpLMClQntlqFc2X1gUj4viwVObKrddH9YDoHvuujAEuNV+bLwFS8XxdSr+Cq3Vf+4F5RgQl6ZR2p1eAzU/HX80YBYyJLCuexwJCO2O1bwCRidAfWBSctswbI12GAJT3yiwFR7+MBjGK2g/WAJR3FdF84E2rK5VR0YH/9k=
parse-2.5.21/tests/number/main.go 0000664 0000000 0000000 00000000240 14117006534 0016615 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import "github.com/tdewolff/parse/v2"
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
_ = parse.Number(data)
return 1
}
parse-2.5.21/tests/replace-entities/ 0000775 0000000 0000000 00000000000 14117006534 0017313 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/replace-entities/corpus/ 0000775 0000000 0000000 00000000000 14117006534 0020626 5 ustar 00root root 0000000 0000000 parse-2.5.21/tests/replace-entities/corpus/1.corpus 0000664 0000000 0000000 00000000017 14117006534 0022221 0 ustar 00root root 0000000 0000000 test&test;test
parse-2.5.21/tests/replace-entities/corpus/2.corpus 0000664 0000000 0000000 00000000022 14117006534 0022216 0 ustar 00root root 0000000 0000000 "";&apos
parse-2.5.21/tests/replace-entities/corpus/3.corpus 0000664 0000000 0000000 00000000016 14117006534 0022222 0 ustar 00root root 0000000 0000000 &test4;&Long;
parse-2.5.21/tests/replace-entities/main.go 0000664 0000000 0000000 00000001031 14117006534 0020561 0 ustar 00root root 0000000 0000000 // +build gofuzz
package fuzz
import "github.com/tdewolff/parse/v2"
// Fuzz is a fuzz test.
func Fuzz(data []byte) int {
data = parse.Copy(data) // ignore const-input error for OSS-Fuzz
newData := parse.ReplaceEntities(data, map[string][]byte{
"test": []byte("&t;"),
"test3": []byte("&test;"),
"test5": []byte(""),
"quot": []byte("\""),
"apos": []byte("'"),
}, map[byte][]byte{
'\'': []byte("""),
'"': []byte("'"),
})
if len(newData) > len(data) {
panic("output longer than input")
}
return 1
}
parse-2.5.21/util.go 0000664 0000000 0000000 00000035531 14117006534 0014227 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"fmt"
"strconv"
"unicode"
)
// Copy returns a copy of the given byte slice.
func Copy(src []byte) (dst []byte) {
dst = make([]byte, len(src))
copy(dst, src)
return
}
// ToLower converts all characters in the byte slice from A-Z to a-z.
func ToLower(src []byte) []byte {
for i, c := range src {
if c >= 'A' && c <= 'Z' {
src[i] = c + ('a' - 'A')
}
}
return src
}
// EqualFold returns true when s matches case-insensitively the targetLower (which must be lowercase).
func EqualFold(s, targetLower []byte) bool {
if len(s) != len(targetLower) {
return false
}
for i, c := range targetLower {
d := s[i]
if d != c && (d < 'A' || d > 'Z' || d+('a'-'A') != c) {
return false
}
}
return true
}
// Printable returns a printable string for given rune
func Printable(r rune) string {
if unicode.IsGraphic(r) {
return fmt.Sprintf("%c", r)
} else if r < 128 {
return fmt.Sprintf("0x%02X", r)
}
return fmt.Sprintf("%U", r)
}
var whitespaceTable = [256]bool{
// ASCII
false, false, false, false, false, false, false, false,
false, true, true, false, true, true, false, false, // tab, new line, form feed, carriage return
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
true, false, false, false, false, false, false, false, // space
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
// non-ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
}
// IsWhitespace returns true for space, \n, \r, \t, \f.
func IsWhitespace(c byte) bool {
return whitespaceTable[c]
}
var newlineTable = [256]bool{
// ASCII
false, false, false, false, false, false, false, false,
false, false, true, false, false, true, false, false, // new line, carriage return
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
// non-ASCII
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
}
// IsNewline returns true for \n, \r.
func IsNewline(c byte) bool {
return newlineTable[c]
}
// IsAllWhitespace returns true when the entire byte slice consists of space, \n, \r, \t, \f.
func IsAllWhitespace(b []byte) bool {
for _, c := range b {
if !IsWhitespace(c) {
return false
}
}
return true
}
// TrimWhitespace removes any leading and trailing whitespace characters.
func TrimWhitespace(b []byte) []byte {
n := len(b)
start := n
for i := 0; i < n; i++ {
if !IsWhitespace(b[i]) {
start = i
break
}
}
end := n
for i := n - 1; i >= start; i-- {
if !IsWhitespace(b[i]) {
end = i + 1
break
}
}
return b[start:end]
}
// ReplaceMultipleWhitespace replaces character series of space, \n, \t, \f, \r into a single space or newline (when the serie contained a \n or \r).
func ReplaceMultipleWhitespace(b []byte) []byte {
j, k := 0, 0 // j is write position, k is start of next text section
for i := 0; i < len(b); i++ {
if IsWhitespace(b[i]) {
start := i
newline := IsNewline(b[i])
i++
for ; i < len(b) && IsWhitespace(b[i]); i++ {
if IsNewline(b[i]) {
newline = true
}
}
if newline {
b[start] = '\n'
} else {
b[start] = ' '
}
if 1 < i-start { // more than one whitespace
if j == 0 {
j = start + 1
} else {
j += copy(b[j:], b[k:start+1])
}
k = i
}
}
}
if j == 0 {
return b
} else if j == 1 { // only if starts with whitespace
b[k-1] = b[0]
return b[k-1:]
} else if k < len(b) {
j += copy(b[j:], b[k:])
}
return b[:j]
}
// replaceEntities will replace in b at index i, assuming that b[i] == '&' and that i+3<len(b). The returned int will be the last character of the entity, so that the next iteration can safely do i++ to continue and not miss any entitites.
func replaceEntities(b []byte, i int, entitiesMap map[string][]byte, revEntitiesMap map[byte][]byte) ([]byte, int) {
const MaxEntityLength = 31 // longest HTML entity: CounterClockwiseContourIntegral
var r []byte
j := i + 1
if b[j] == '#' {
j++
if b[j] == 'x' {
j++
c := 0
for ; j < len(b) && (b[j] >= '0' && b[j] <= '9' || b[j] >= 'a' && b[j] <= 'f' || b[j] >= 'A' && b[j] <= 'F'); j++ {
if b[j] <= '9' {
c = c<<4 + int(b[j]-'0')
} else if b[j] <= 'F' {
c = c<<4 + int(b[j]-'A') + 10
} else if b[j] <= 'f' {
c = c<<4 + int(b[j]-'a') + 10
}
}
if j <= i+3 || 10000 <= c {
return b, j - 1
}
if c < 128 {
r = []byte{byte(c)}
} else {
r = append(r, '&', '#')
r = strconv.AppendInt(r, int64(c), 10)
r = append(r, ';')
}
} else {
c := 0
for ; j < len(b) && c < 128 && b[j] >= '0' && b[j] <= '9'; j++ {
c = c*10 + int(b[j]-'0')
}
if j <= i+2 || 128 <= c {
return b, j - 1
}
r = []byte{byte(c)}
}
} else {
for ; j < len(b) && j-i-1 <= MaxEntityLength && b[j] != ';'; j++ {
}
if j <= i+1 || len(b) <= j {
return b, j - 1
}
var ok bool
r, ok = entitiesMap[string(b[i+1:j])]
if !ok {
return b, j
}
}
// j is at semicolon
n := j + 1 - i
if j < len(b) && b[j] == ';' && 2 < n {
if len(r) == 1 {
if q, ok := revEntitiesMap[r[0]]; ok {
if len(q) == len(b[i:j+1]) && bytes.Equal(q, b[i:j+1]) {
return b, j
}
r = q
} else if r[0] == '&' {
// check if for example & is followed by something that could potentially be an entity
k := j + 1
if k < len(b) && b[k] == '#' {
k++
}
for ; k < len(b) && k-j <= MaxEntityLength && (b[k] >= '0' && b[k] <= '9' || b[k] >= 'a' && b[k] <= 'z' || b[k] >= 'A' && b[k] <= 'Z'); k++ {
}
if k < len(b) && b[k] == ';' {
return b, k
}
}
}
copy(b[i:], r)
copy(b[i+len(r):], b[j+1:])
b = b[:len(b)-n+len(r)]
return b, i + len(r) - 1
}
return b, i
}
// ReplaceEntities replaces all occurrences of entites (such as ") to their respective unencoded bytes.
func ReplaceEntities(b []byte, entitiesMap map[string][]byte, revEntitiesMap map[byte][]byte) []byte {
for i := 0; i < len(b); i++ {
if b[i] == '&' && i+3 < len(b) {
b, i = replaceEntities(b, i, entitiesMap, revEntitiesMap)
}
}
return b
}
// ReplaceMultipleWhitespaceAndEntities is a combination of ReplaceMultipleWhitespace and ReplaceEntities. It is faster than executing both sequentially.
func ReplaceMultipleWhitespaceAndEntities(b []byte, entitiesMap map[string][]byte, revEntitiesMap map[byte][]byte) []byte {
j, k := 0, 0 // j is write position, k is start of next text section
for i := 0; i < len(b); i++ {
if IsWhitespace(b[i]) {
start := i
newline := IsNewline(b[i])
i++
for ; i < len(b) && IsWhitespace(b[i]); i++ {
if IsNewline(b[i]) {
newline = true
}
}
if newline {
b[start] = '\n'
} else {
b[start] = ' '
}
if 1 < i-start { // more than one whitespace
if j == 0 {
j = start + 1
} else {
j += copy(b[j:], b[k:start+1])
}
k = i
}
}
if i+3 < len(b) && b[i] == '&' {
b, i = replaceEntities(b, i, entitiesMap, revEntitiesMap)
}
}
if j == 0 {
return b
} else if j == 1 { // only if starts with whitespace
b[k-1] = b[0]
return b[k-1:]
} else if k < len(b) {
j += copy(b[j:], b[k:])
}
return b[:j]
}
// URLEncodingTable is a charmap for which characters need escaping in the URL encoding scheme
var URLEncodingTable = [256]bool{
// ASCII
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, false, true, true, true, true, true, false, // space, ", #, $, %, &
false, false, false, true, true, false, false, true, // +, comma, /
false, false, false, false, false, false, false, false,
false, false, true, true, true, true, true, true, // :, ;, <, =, >, ?
true, false, false, false, false, false, false, false, // @
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, true, true, true, true, false, // [, \, ], ^
true, false, false, false, false, false, false, false, // `
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, true, true, true, false, true, // {, |, }, DEL
// non-ASCII
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
}
// DataURIEncodingTable is a charmap for which characters need escaping in the Data URI encoding scheme
// Escape only non-printable characters, unicode and %, #, &. IE11 additionally requires encoding of
// \, [, ], ", <, >, `, {, }, |, ^ which is not required by Chrome, Firefox, Opera, Edge, Safari, Yandex
var DataURIEncodingTable = [256]bool{
// ASCII
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
false, false, true, true, false, true, true, false, // ", #, %, &
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, true, false, true, false, // <, >
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, true, true, true, true, false, // [, \, ], ^
true, false, false, false, false, false, false, false, // `
false, false, false, false, false, false, false, false,
false, false, false, false, false, false, false, false,
false, false, false, true, true, true, false, true, // {, |, }, DEL
// non-ASCII
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
true, true, true, true, true, true, true, true,
}
// EncodeURL encodes bytes using the URL encoding scheme
func EncodeURL(b []byte, table [256]bool) []byte {
for i := 0; i < len(b); i++ {
c := b[i]
if table[c] {
if c == ' ' {
b[i] = '+'
} else {
b = append(b, 0, 0)
copy(b[i+3:], b[i+1:])
b[i+0] = '%'
b[i+1] = "0123456789ABCDEF"[c>>4]
b[i+2] = "0123456789ABCDEF"[c&15]
}
}
}
return b
}
// DecodeURL decodes an URL encoded using the URL encoding scheme
func DecodeURL(b []byte) []byte {
for i := 0; i < len(b); i++ {
if b[i] == '%' && i+2 < len(b) {
j := i + 1
c := 0
for ; j < i+3 && (b[j] >= '0' && b[j] <= '9' || b[j] >= 'a' && b[j] <= 'f' || b[j] >= 'A' && b[j] <= 'F'); j++ {
if b[j] <= '9' {
c = c<<4 + int(b[j]-'0')
} else if b[j] <= 'F' {
c = c<<4 + int(b[j]-'A') + 10
} else if b[j] <= 'f' {
c = c<<4 + int(b[j]-'a') + 10
}
}
if j == i+3 && c < 128 {
b[i] = byte(c)
b = append(b[:i+1], b[i+3:]...)
}
} else if b[i] == '+' {
b[i] = ' '
}
}
return b
}
parse-2.5.21/util_test.go 0000664 0000000 0000000 00000022511 14117006534 0015260 0 ustar 00root root 0000000 0000000 package parse
import (
"bytes"
"math/rand"
"net/url"
"regexp"
"testing"
"github.com/tdewolff/test"
)
func helperRandChars(n, m int, chars string) [][]byte {
r := make([][]byte, n)
for i := range r {
for j := 0; j < m; j++ {
r[i] = append(r[i], chars[rand.Intn(len(chars))])
}
}
return r
}
func helperRandStrings(n, m int, ss []string) [][]byte {
r := make([][]byte, n)
for i := range r {
for j := 0; j < m; j++ {
r[i] = append(r[i], []byte(ss[rand.Intn(len(ss))])...)
}
}
return r
}
////////////////////////////////////////////////////////////////
var wsSlices [][]byte
var entitySlices [][]byte
var encodedURLSlices [][]byte
var urlSlices [][]byte
func init() {
wsSlices = helperRandChars(10000, 50, "abcdefg \n\r\f\t")
entitySlices = helperRandStrings(100, 5, []string{""", "'", "'", " ", " ", "test"})
encodedURLSlices = helperRandStrings(100, 5, []string{"%20", "%3D", "test"})
urlSlices = helperRandStrings(100, 5, []string{"~", "\"", "<", "test"})
}
func TestCopy(t *testing.T) {
foo := []byte("abc")
bar := Copy(foo)
foo[0] = 'b'
test.String(t, string(foo), "bbc")
test.String(t, string(bar), "abc")
}
func TestToLower(t *testing.T) {
foo := []byte("Abc")
bar := ToLower(foo)
bar[1] = 'B'
test.String(t, string(foo), "aBc")
test.String(t, string(bar), "aBc")
}
func TestEqualFold(t *testing.T) {
test.That(t, EqualFold([]byte("Abc"), []byte("abc")))
test.That(t, !EqualFold([]byte("Abcd"), []byte("abc")))
test.That(t, !EqualFold([]byte("Bbc"), []byte("abc")))
test.That(t, !EqualFold([]byte("[]"), []byte("{}"))) // same distance in ASCII as 'a' and 'A'
}
func TestWhitespace(t *testing.T) {
test.That(t, IsAllWhitespace([]byte("\t \r\n\f")))
test.That(t, !IsAllWhitespace([]byte("\t \r\n\fx")))
}
func TestTrim(t *testing.T) {
test.Bytes(t, TrimWhitespace([]byte("a")), []byte("a"))
test.Bytes(t, TrimWhitespace([]byte(" a")), []byte("a"))
test.Bytes(t, TrimWhitespace([]byte("a ")), []byte("a"))
test.Bytes(t, TrimWhitespace([]byte(" ")), []byte(""))
}
func TestReplaceMultipleWhitespace(t *testing.T) {
test.Bytes(t, ReplaceMultipleWhitespace([]byte(" a")), []byte(" a"))
test.Bytes(t, ReplaceMultipleWhitespace([]byte("a ")), []byte("a "))
test.Bytes(t, ReplaceMultipleWhitespace([]byte("a b ")), []byte("a b "))
test.Bytes(t, ReplaceMultipleWhitespace([]byte(" a b ")), []byte(" a b "))
test.Bytes(t, ReplaceMultipleWhitespace([]byte(" a b ")), []byte(" a b "))
test.Bytes(t, ReplaceMultipleWhitespace([]byte(" a b ")), []byte(" a b "))
test.Bytes(t, ReplaceMultipleWhitespace([]byte(" a")), []byte(" a"))
test.Bytes(t, ReplaceMultipleWhitespace([]byte("a b")), []byte("a b"))
}
func TestReplaceMultipleWhitespaceRandom(t *testing.T) {
wsRegexp := regexp.MustCompile("[ \t\f]+")
wsNewlinesRegexp := regexp.MustCompile("[ ]*[\r\n][ \r\n]*")
for _, e := range wsSlices {
reference := wsRegexp.ReplaceAll(e, []byte(" "))
reference = wsNewlinesRegexp.ReplaceAll(reference, []byte("\n"))
test.Bytes(t, ReplaceMultipleWhitespace(Copy(e)), reference, "in '"+string(e)+"'")
}
}
func TestReplaceEntities(t *testing.T) {
entitiesMap := map[string][]byte{
"varphi": []byte("ϕ"),
"varpi": []byte("ϖ"),
"quot": []byte("\""),
"apos": []byte("'"),
"amp": []byte("&"),
}
revEntitiesMap := map[byte][]byte{
'\'': []byte("'"),
}
var entityTests = []struct {
entity string
expected string
}{
{""", `"`},
{"'", `'`},
{""", `"`},
{"'", `'`},
{" ", ` `},
{""", `"`},
{"'", `'`},
{"⏧", `⏧`},
{"⏧", `⏧`},
{"⏧", `⏧`},
{"⏧", `⏧`},
{"✏", `✏`},
{"✐", `✐`},
{"'"", `'"`},
{""", `"`},
{""", `"`},
{"&apos", `&apos`},
{"&", `&`},
{"'", `'`},
{"&amp;", `&amp;`},
{"&#34;", `&#34;`},
{"&a mp;", `&a mp;`},
{"&DiacriticalAcute;", `&DiacriticalAcute;`},
{"&CounterClockwiseContourIntegral;", `&CounterClockwiseContourIntegral;`},
{"&CounterClockwiseContourIntegralL;", `&CounterClockwiseContourIntegralL;`},
{"ϕ", "ϕ"},
{"ϖ", "ϖ"},
{"&varnone;", "&varnone;"},
}
for _, tt := range entityTests {
t.Run(tt.entity, func(t *testing.T) {
b := ReplaceEntities([]byte(tt.entity), entitiesMap, revEntitiesMap)
test.T(t, string(b), tt.expected, "in '"+tt.entity+"'")
})
}
}
func TestReplaceEntitiesRandom(t *testing.T) {
entitiesMap := map[string][]byte{
"quot": []byte("\""),
"apos": []byte("'"),
}
revEntitiesMap := map[byte][]byte{
'\'': []byte("'"),
}
quotRegexp := regexp.MustCompile(""")
aposRegexp := regexp.MustCompile("('|')")
for _, e := range entitySlices {
reference := quotRegexp.ReplaceAll(e, []byte("\""))
reference = aposRegexp.ReplaceAll(reference, []byte("'"))
test.Bytes(t, ReplaceEntities(Copy(e), entitiesMap, revEntitiesMap), reference, "in '"+string(e)+"'")
}
}
func TestReplaceMultipleWhitespaceAndEntities(t *testing.T) {
entitiesMap := map[string][]byte{
"varphi": []byte("ϕ"),
}
var entityTests = []struct {
entity string
expected string
}{
{" ϕ " \n ", " ϕ \"\n"},
}
for _, tt := range entityTests {
t.Run(tt.entity, func(t *testing.T) {
b := ReplaceMultipleWhitespaceAndEntities([]byte(tt.entity), entitiesMap, nil)
test.T(t, string(b), tt.expected, "in '"+tt.entity+"'")
})
}
}
func TestReplaceMultipleWhitespaceAndEntitiesRandom(t *testing.T) {
entitiesMap := map[string][]byte{
"quot": []byte("\""),
"apos": []byte("'"),
}
revEntitiesMap := map[byte][]byte{
'\'': []byte("'"),
}
wsRegexp := regexp.MustCompile("[ ]+")
quotRegexp := regexp.MustCompile(""")
aposRegexp := regexp.MustCompile("('|')")
for _, e := range entitySlices {
reference := wsRegexp.ReplaceAll(e, []byte(" "))
reference = quotRegexp.ReplaceAll(reference, []byte("\""))
reference = aposRegexp.ReplaceAll(reference, []byte("'"))
test.Bytes(t, ReplaceMultipleWhitespaceAndEntities(Copy(e), entitiesMap, revEntitiesMap), reference, "in '"+string(e)+"'")
}
}
func TestPrintable(t *testing.T) {
var tests = []struct {
s string
printable string
}{
{"a", "a"},
{"\x00", "0x00"},
{"\x7F", "0x7F"},
{"\u0800", "ࠀ"},
{"\u200F", "U+200F"},
}
for _, tt := range tests {
t.Run(tt.s, func(t *testing.T) {
printable := ""
for _, r := range tt.s {
printable += Printable(r)
}
test.T(t, printable, tt.printable)
})
}
}
func TestDecodeURL(t *testing.T) {
var urlTests = []struct {
url string
expected string
}{
{"%20%3F%7E", " ?~"},
{"%80", "%80"},
{"%2B%2b", "++"},
{"%' ", "%' "},
{"a+b", "a b"},
}
for _, tt := range urlTests {
t.Run(tt.url, func(t *testing.T) {
b := DecodeURL([]byte(tt.url))
test.T(t, string(b), tt.expected, "in '"+tt.url+"'")
})
}
}
func TestDecodeURLRandom(t *testing.T) {
for _, e := range encodedURLSlices {
reference, _ := url.QueryUnescape(string(e))
test.Bytes(t, DecodeURL(Copy(e)), []byte(reference), "in '"+string(e)+"'")
}
}
func TestEncodeURL(t *testing.T) {
var urlTests = []struct {
url string
expected string
}{
{"AZaz09-_.!~*'()", "AZaz09-_.!~*'()"},
{"<>", "%3C%3E"},
{"\u2318", "%E2%8C%98"},
{"a b", "a+b"},
}
for _, tt := range urlTests {
t.Run(tt.url, func(t *testing.T) {
b := EncodeURL([]byte(tt.url), URLEncodingTable)
test.T(t, string(b), tt.expected, "in '"+tt.url+"'")
})
}
}
func TestEncodeURLRandom(t *testing.T) {
for _, e := range urlSlices {
reference := url.QueryEscape(string(e))
test.Bytes(t, EncodeURL(Copy(e), URLEncodingTable), []byte(reference), "in '"+string(e)+"'")
}
}
////////////////////////////////////////////////////////////////
func BenchmarkBytesTrim(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
bytes.TrimSpace(e)
}
}
}
func BenchmarkTrim(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
TrimWhitespace(e)
}
}
}
func BenchmarkReplaceMultipleWhitespace(b *testing.B) {
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
ReplaceMultipleWhitespace(e)
}
}
}
func BenchmarkWhitespaceTable(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if IsWhitespace(c) {
n++
}
}
}
}
}
func BenchmarkWhitespaceIf1(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if c == ' ' {
n++
}
}
}
}
}
func BenchmarkWhitespaceIf2(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if c == ' ' || c == '\n' {
n++
}
}
}
}
}
func BenchmarkWhitespaceIf3(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if c == ' ' || c == '\n' || c == '\r' {
n++
}
}
}
}
}
func BenchmarkWhitespaceIf4(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if c == ' ' || c == '\n' || c == '\r' || c == '\t' {
n++
}
}
}
}
}
func BenchmarkWhitespaceIf5(b *testing.B) {
n := 0
for i := 0; i < b.N; i++ {
for _, e := range wsSlices {
for _, c := range e {
if c == ' ' || c == '\n' || c == '\r' || c == '\t' || c == '\f' {
n++
}
}
}
}
}
parse-2.5.21/xml/ 0000775 0000000 0000000 00000000000 14117006534 0013514 5 ustar 00root root 0000000 0000000 parse-2.5.21/xml/README.md 0000664 0000000 0000000 00000004070 14117006534 0014774 0 ustar 00root root 0000000 0000000 # XML [](https://pkg.go.dev/github.com/tdewolff/parse/v2/xml?tab=doc)
This package is an XML lexer written in [Go][1]. It follows the specification at [Extensible Markup Language (XML) 1.0 (Fifth Edition)](http://www.w3.org/TR/REC-xml/). The lexer takes an io.Reader and converts it into tokens until the EOF.
## Installation
Run the following command
go get -u github.com/tdewolff/parse/v2/xml
or add the following import and run project with `go get`
import "github.com/tdewolff/parse/v2/xml"
## Lexer
### Usage
The following initializes a new Lexer with io.Reader `r`:
``` go
l := xml.NewLexer(parse.NewInput(r))
```
To tokenize until EOF an error, use:
``` go
for {
tt, data := l.Next()
switch tt {
case xml.ErrorToken:
// error or EOF set in l.Err()
return
case xml.StartTagToken:
// ...
for {
ttAttr, dataAttr := l.Next()
if ttAttr != xml.AttributeToken {
// handle StartTagCloseToken/StartTagCloseVoidToken/StartTagClosePIToken
break
}
// ...
}
case xml.EndTagToken:
// ...
}
}
```
All tokens:
``` go
ErrorToken TokenType = iota // extra token when errors occur
CommentToken
CDATAToken
StartTagToken
StartTagCloseToken
StartTagCloseVoidToken
StartTagClosePIToken
EndTagToken
AttributeToken
TextToken
```
### Examples
``` go
package main
import (
"os"
"github.com/tdewolff/parse/v2/xml"
)
// Tokenize XML from stdin.
func main() {
l := xml.NewLexer(parse.NewInput(os.Stdin))
for {
tt, data := l.Next()
switch tt {
case xml.ErrorToken:
if l.Err() != io.EOF {
fmt.Println("Error on line", l.Line(), ":", l.Err())
}
return
case xml.StartTagToken:
fmt.Println("Tag", string(data))
for {
ttAttr, dataAttr := l.Next()
if ttAttr != xml.AttributeToken {
break
}
key := dataAttr
val := l.AttrVal()
fmt.Println("Attribute", string(key), "=", string(val))
}
// ...
}
}
}
```
## License
Released under the [MIT license](https://github.com/tdewolff/parse/blob/master/LICENSE.md).
[1]: http://golang.org/ "Go Language"
parse-2.5.21/xml/lex.go 0000664 0000000 0000000 00000016540 14117006534 0014641 0 ustar 00root root 0000000 0000000 // Package xml is an XML1.0 lexer following the specifications at http://www.w3.org/TR/xml/.
package xml
import (
"strconv"
"github.com/tdewolff/parse/v2"
)
// TokenType determines the type of token, eg. a number or a semicolon.
type TokenType uint32
// TokenType values.
const (
ErrorToken TokenType = iota // extra token when errors occur
CommentToken
DOCTYPEToken
CDATAToken
StartTagToken
StartTagPIToken
StartTagCloseToken
StartTagCloseVoidToken
StartTagClosePIToken
EndTagToken
AttributeToken
TextToken
)
// String returns the string representation of a TokenType.
func (tt TokenType) String() string {
switch tt {
case ErrorToken:
return "Error"
case CommentToken:
return "Comment"
case DOCTYPEToken:
return "DOCTYPE"
case CDATAToken:
return "CDATA"
case StartTagToken:
return "StartTag"
case StartTagPIToken:
return "StartTagPI"
case StartTagCloseToken:
return "StartTagClose"
case StartTagCloseVoidToken:
return "StartTagCloseVoid"
case StartTagClosePIToken:
return "StartTagClosePI"
case EndTagToken:
return "EndTag"
case AttributeToken:
return "Attribute"
case TextToken:
return "Text"
}
return "Invalid(" + strconv.Itoa(int(tt)) + ")"
}
////////////////////////////////////////////////////////////////
// Lexer is the state for the lexer.
type Lexer struct {
r *parse.Input
err error
inTag bool
text []byte
attrVal []byte
}
// NewLexer returns a new Lexer for a given io.Reader.
func NewLexer(r *parse.Input) *Lexer {
return &Lexer{
r: r,
}
}
// Err returns the error encountered during lexing, this is often io.EOF but also other errors can be returned.
func (l *Lexer) Err() error {
if l.err != nil {
return l.err
}
return l.r.Err()
}
// Text returns the textual representation of a token. This excludes delimiters and additional leading/trailing characters.
func (l *Lexer) Text() []byte {
return l.text
}
// AttrVal returns the attribute value when an AttributeToken was returned from Next.
func (l *Lexer) AttrVal() []byte {
return l.attrVal
}
// Next returns the next Token. It returns ErrorToken when an error was encountered. Using Err() one can retrieve the error message.
func (l *Lexer) Next() (TokenType, []byte) {
l.text = nil
var c byte
if l.inTag {
l.attrVal = nil
for { // before attribute name state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' {
l.r.Move(1)
continue
}
break
}
if c == 0 {
if l.r.Err() == nil {
l.err = parse.NewErrorLexer(l.r, "XML parse error: unexpected NULL character")
}
return ErrorToken, nil
} else if c != '>' && (c != '/' && c != '?' || l.r.Peek(1) != '>') {
return AttributeToken, l.shiftAttribute()
}
l.r.Skip()
l.inTag = false
if c == '/' {
l.r.Move(2)
return StartTagCloseVoidToken, l.r.Shift()
} else if c == '?' {
l.r.Move(2)
return StartTagClosePIToken, l.r.Shift()
} else {
l.r.Move(1)
return StartTagCloseToken, l.r.Shift()
}
}
for {
c = l.r.Peek(0)
if c == '<' {
if l.r.Pos() > 0 {
l.text = l.r.Shift()
return TextToken, l.text
}
c = l.r.Peek(1)
if c == '/' {
l.r.Move(2)
return EndTagToken, l.shiftEndTag()
} else if c == '!' {
l.r.Move(2)
if l.at('-', '-') {
l.r.Move(2)
return CommentToken, l.shiftCommentText()
} else if l.at('[', 'C', 'D', 'A', 'T', 'A', '[') {
l.r.Move(7)
return CDATAToken, l.shiftCDATAText()
} else if l.at('D', 'O', 'C', 'T', 'Y', 'P', 'E') {
l.r.Move(7)
return DOCTYPEToken, l.shiftDOCTYPEText()
}
l.r.Move(-2)
} else if c == '?' {
l.r.Move(2)
l.inTag = true
return StartTagPIToken, l.shiftStartTag()
}
l.r.Move(1)
l.inTag = true
return StartTagToken, l.shiftStartTag()
} else if c == 0 {
if l.r.Pos() > 0 {
l.text = l.r.Shift()
return TextToken, l.text
}
if l.r.Err() == nil {
l.err = parse.NewErrorLexer(l.r, "XML parse error: unexpected NULL character")
}
return ErrorToken, nil
}
l.r.Move(1)
}
}
////////////////////////////////////////////////////////////////
// The following functions follow the specifications at http://www.w3.org/html/wg/drafts/html/master/syntax.html
func (l *Lexer) shiftDOCTYPEText() []byte {
inString := false
inBrackets := false
for {
c := l.r.Peek(0)
if c == '"' {
inString = !inString
} else if (c == '[' || c == ']') && !inString {
inBrackets = (c == '[')
} else if c == '>' && !inString && !inBrackets {
l.text = l.r.Lexeme()[9:]
l.r.Move(1)
return l.r.Shift()
} else if c == 0 {
l.text = l.r.Lexeme()[9:]
return l.r.Shift()
}
l.r.Move(1)
}
}
func (l *Lexer) shiftCDATAText() []byte {
for {
c := l.r.Peek(0)
if c == ']' && l.r.Peek(1) == ']' && l.r.Peek(2) == '>' {
l.text = l.r.Lexeme()[9:]
l.r.Move(3)
return l.r.Shift()
} else if c == 0 {
l.text = l.r.Lexeme()[9:]
return l.r.Shift()
}
l.r.Move(1)
}
}
func (l *Lexer) shiftCommentText() []byte {
for {
c := l.r.Peek(0)
if c == '-' && l.r.Peek(1) == '-' && l.r.Peek(2) == '>' {
l.text = l.r.Lexeme()[4:]
l.r.Move(3)
return l.r.Shift()
} else if c == 0 {
return l.r.Shift()
}
l.r.Move(1)
}
}
func (l *Lexer) shiftStartTag() []byte {
nameStart := l.r.Pos()
for {
if c := l.r.Peek(0); c == ' ' || c == '>' || (c == '/' || c == '?') && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == 0 {
break
}
l.r.Move(1)
}
l.text = l.r.Lexeme()[nameStart:]
return l.r.Shift()
}
func (l *Lexer) shiftAttribute() []byte {
nameStart := l.r.Pos()
var c byte
for { // attribute name state
if c = l.r.Peek(0); c == ' ' || c == '=' || c == '>' || (c == '/' || c == '?') && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == 0 {
break
}
l.r.Move(1)
}
nameEnd := l.r.Pos()
for { // after attribute name state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' {
l.r.Move(1)
continue
}
break
}
if c == '=' {
l.r.Move(1)
for { // before attribute value state
if c = l.r.Peek(0); c == ' ' || c == '\t' || c == '\n' || c == '\r' {
l.r.Move(1)
continue
}
break
}
attrPos := l.r.Pos()
delim := c
if delim == '"' || delim == '\'' { // attribute value single- and double-quoted state
l.r.Move(1)
for {
c = l.r.Peek(0)
if c == delim {
l.r.Move(1)
break
} else if c == 0 {
break
}
l.r.Move(1)
if c == '\t' || c == '\n' || c == '\r' {
l.r.Lexeme()[l.r.Pos()-1] = ' '
}
}
} else { // attribute value unquoted state
for {
if c = l.r.Peek(0); c == ' ' || c == '>' || (c == '/' || c == '?') && l.r.Peek(1) == '>' || c == '\t' || c == '\n' || c == '\r' || c == 0 {
break
}
l.r.Move(1)
}
}
l.attrVal = l.r.Lexeme()[attrPos:]
} else {
l.r.Rewind(nameEnd)
l.attrVal = nil
}
l.text = l.r.Lexeme()[nameStart:nameEnd]
return l.r.Shift()
}
func (l *Lexer) shiftEndTag() []byte {
for {
c := l.r.Peek(0)
if c == '>' {
l.text = l.r.Lexeme()[2:]
l.r.Move(1)
break
} else if c == 0 {
l.text = l.r.Lexeme()[2:]
break
}
l.r.Move(1)
}
end := len(l.text)
for end > 0 {
if c := l.text[end-1]; c == ' ' || c == '\t' || c == '\n' || c == '\r' {
end--
continue
}
break
}
l.text = l.text[:end]
return l.r.Shift()
}
////////////////////////////////////////////////////////////////
func (l *Lexer) at(b ...byte) bool {
for i, c := range b {
if l.r.Peek(i) != c {
return false
}
}
return true
}
parse-2.5.21/xml/lex_test.go 0000664 0000000 0000000 00000017374 14117006534 0015706 0 ustar 00root root 0000000 0000000 package xml
import (
"fmt"
"io"
"testing"
"github.com/tdewolff/parse/v2"
"github.com/tdewolff/test"
)
type TTs []TokenType
func TestTokens(t *testing.T) {
var tokenTests = []struct {
xml string
expected []TokenType
}{
{"", TTs{}},
{"<!-- comment -->", TTs{CommentToken}},
{"<!-- comment \n multi \r line -->", TTs{CommentToken}},
{"<foo/>", TTs{StartTagToken, StartTagCloseVoidToken}},
{"<foo \t\r\n/>", TTs{StartTagToken, StartTagCloseVoidToken}},
{"<foo:bar.qux-norf/>", TTs{StartTagToken, StartTagCloseVoidToken}},
{"<foo></foo>", TTs{StartTagToken, StartTagCloseToken, EndTagToken}},
{"<foo>text</foo>", TTs{StartTagToken, StartTagCloseToken, TextToken, EndTagToken}},
{"<foo/> text", TTs{StartTagToken, StartTagCloseVoidToken, TextToken}},
{"<a> <b> <c>text</c> </b> </a>", TTs{StartTagToken, StartTagCloseToken, TextToken, StartTagToken, StartTagCloseToken, TextToken, StartTagToken, StartTagCloseToken, TextToken, EndTagToken, TextToken, EndTagToken, TextToken, EndTagToken}},
{"<foo a='a' b=\"b\" c=c/>", TTs{StartTagToken, AttributeToken, AttributeToken, AttributeToken, StartTagCloseVoidToken}},
{"<foo a=\"\"/>", TTs{StartTagToken, AttributeToken, StartTagCloseVoidToken}},
{"<foo a-b=\"\"/>", TTs{StartTagToken, AttributeToken, StartTagCloseVoidToken}},
{"<foo \nchecked \r\n value\r=\t'=/>\"' />", TTs{StartTagToken, AttributeToken, AttributeToken, StartTagCloseVoidToken}},
{"<?xml?>", TTs{StartTagPIToken, StartTagClosePIToken}},
{"<?xml a=\"a\" ?>", TTs{StartTagPIToken, AttributeToken, StartTagClosePIToken}},
{"<?xml a=a?>", TTs{StartTagPIToken, AttributeToken, StartTagClosePIToken}},
{"<![CDATA[ test ]]>", TTs{CDATAToken}},
{"<!DOCTYPE>", TTs{DOCTYPEToken}},
{"<!DOCTYPE note SYSTEM \"Note.dtd\">", TTs{DOCTYPEToken}},
{`<!DOCTYPE note [<!ENTITY nbsp " "><!ENTITY writer "Writer: Donald Duck."><!ENTITY copyright "Copyright:]> W3Schools.">]>`, TTs{DOCTYPEToken}},
{"<!foo>", TTs{StartTagToken, StartTagCloseToken}},
// early endings
{"<!-- comment", TTs{CommentToken}},
{"<foo", TTs{StartTagToken}},
{"</foo", TTs{EndTagToken}},
{"<foo x", TTs{StartTagToken, AttributeToken}},
{"<foo x=", TTs{StartTagToken, AttributeToken}},
{"<foo x='", TTs{StartTagToken, AttributeToken}},
{"<foo x=''", TTs{StartTagToken, AttributeToken}},
{"<?xml", TTs{StartTagPIToken}},
{"<![CDATA[ test", TTs{CDATAToken}},
{"<!DOCTYPE note SYSTEM", TTs{DOCTYPEToken}},
// go fuzz
{"</", TTs{EndTagToken}},
{"</\n", TTs{EndTagToken}},
}
for _, tt := range tokenTests {
t.Run(tt.xml, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.xml))
i := 0
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
}
test.That(t, i < len(tt.expected), "index", i, "must not exceed expected token types size", len(tt.expected))
if i < len(tt.expected) {
test.T(t, token, tt.expected[i], "token types must match")
}
i++
}
})
}
// coverage
for i := 0; ; i++ {
if TokenType(i).String() == fmt.Sprintf("Invalid(%d)", i) {
break
}
}
}
func TestTags(t *testing.T) {
var tagTests = []struct {
xml string
expected string
}{
{"<foo:bar.qux-norf/>", "foo:bar.qux-norf"},
{"<?xml?>", "xml"},
{"<foo?bar/qux>", "foo?bar/qux"},
{"<!DOCTYPE note SYSTEM \"Note.dtd\">", " note SYSTEM \"Note.dtd\""},
// early endings
{"<foo ", "foo"},
}
for _, tt := range tagTests {
t.Run(tt.xml, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.xml))
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.Fail(t, "when error occurred we must be at the end")
break
} else if token == StartTagToken || token == StartTagPIToken || token == EndTagToken || token == DOCTYPEToken {
test.String(t, string(l.Text()), tt.expected, "tags must match")
break
}
}
})
}
}
func TestAttributes(t *testing.T) {
var attributeTests = []struct {
attr string
expected []string
}{
{"<foo a=\"b\" />", []string{"a", "\"b\""}},
{"<foo \nchecked \r\n value\r=\t'=/>\"' />", []string{"checked", "", "value", "'=/>\"'"}},
{"<foo bar=\" a \n\t\r b \" />", []string{"bar", "\" a b \""}},
{"<?xml a=b?>", []string{"a", "b"}},
{"<foo /=? >", []string{"/", "?"}},
// early endings
{"<foo x", []string{"x", ""}},
{"<foo x=", []string{"x", ""}},
{"<foo x='", []string{"x", "'"}},
}
for _, tt := range attributeTests {
t.Run(tt.attr, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.attr))
i := 0
for {
token, _ := l.Next()
if token == ErrorToken {
test.T(t, l.Err(), io.EOF)
test.T(t, i, len(tt.expected), "when error occurred we must be at the end")
break
} else if token == AttributeToken {
test.That(t, i+1 < len(tt.expected), "index", i+1, "must not exceed expected attributes size", len(tt.expected))
if i+1 < len(tt.expected) {
test.String(t, string(l.Text()), tt.expected[i], "attribute keys must match")
test.String(t, string(l.AttrVal()), tt.expected[i+1], "attribute keys must match")
i += 2
}
}
}
})
}
}
func TestErrors(t *testing.T) {
var errorTests = []struct {
xml string
col int
}{
{"a\x00b", 2},
{"<\x00 b='5'>", 2},
{"<a\x00b='5'>", 3},
{"<a \x00='5'>", 4},
{"<a b\x00'5'>", 5},
{"<a b=\x005'>", 6},
{"<a b='\x00'>", 7},
{"<a b='5\x00>", 8},
{"<a b='5'\x00", 9},
{"</\x00a>", 3},
{"</ \x00>", 4},
{"</ a\x00", 5},
{"<!\x00", 3},
{"<![CDATA[\x00", 10},
{"/*\x00", 3},
}
for _, tt := range errorTests {
t.Run(tt.xml, func(t *testing.T) {
l := NewLexer(parse.NewInputString(tt.xml))
for {
token, _ := l.Next()
if token == ErrorToken {
if perr, ok := l.Err().(*parse.Error); ok {
_, col, _ := perr.Position()
test.T(t, col, tt.col)
} else {
test.Fail(t, "bad error:", l.Err())
}
break
}
}
})
}
}
func TestTextAndAttrVal(t *testing.T) {
l := NewLexer(parse.NewInputString(`<xml attr="val" >text<!--comment--><!DOCTYPE doctype><![CDATA[cdata]]>`))
_, data := l.Next()
test.Bytes(t, data, []byte("<xml"))
test.Bytes(t, l.Text(), []byte("xml"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte(` attr="val"`))
test.Bytes(t, l.Text(), []byte("attr"))
test.Bytes(t, l.AttrVal(), []byte(`"val"`))
_, data = l.Next()
test.Bytes(t, data, []byte(">"))
test.Bytes(t, l.Text(), nil)
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("text"))
test.Bytes(t, l.Text(), []byte("text"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<!--comment-->"))
test.Bytes(t, l.Text(), []byte("comment"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<!DOCTYPE doctype>"))
test.Bytes(t, l.Text(), []byte(" doctype"))
test.Bytes(t, l.AttrVal(), nil)
_, data = l.Next()
test.Bytes(t, data, []byte("<![CDATA[cdata]]>"))
test.Bytes(t, l.Text(), []byte("cdata"))
test.Bytes(t, l.AttrVal(), nil)
}
func TestOffset(t *testing.T) {
z := parse.NewInputString(`<div attr="val">text</div>`)
l := NewLexer(z)
test.T(t, z.Offset(), 0)
_, _ = l.Next()
test.T(t, z.Offset(), 4) // <div
_, _ = l.Next()
test.T(t, z.Offset(), 15) // attr="val"
_, _ = l.Next()
test.T(t, z.Offset(), 16) // >
_, _ = l.Next()
test.T(t, z.Offset(), 20) // text
_, _ = l.Next()
test.T(t, z.Offset(), 26) // </div>
}
////////////////////////////////////////////////////////////////
func ExampleNewLexer() {
l := NewLexer(parse.NewInputString("<span class='user'>John Doe</span>"))
out := ""
for {
tt, data := l.Next()
if tt == ErrorToken {
break
}
out += string(data)
}
fmt.Println(out)
// Output: <span class='user'>John Doe</span>
}
parse-2.5.21/xml/util.go 0000664 0000000 0000000 00000003322 14117006534 0015020 0 ustar 00root root 0000000 0000000 package xml
var (
ltEntityBytes = []byte("<")
ampEntityBytes = []byte("&")
singleQuoteEntityBytes = []byte("'")
doubleQuoteEntityBytes = []byte(""")
)
// EscapeAttrVal returns the escape attribute value bytes without quotes.
func EscapeAttrVal(buf *[]byte, b []byte) []byte {
singles := 0
doubles := 0
for _, c := range b {
if c == '"' {
doubles++
} else if c == '\'' {
singles++
}
}
n := len(b) + 2
var quote byte
var escapedQuote []byte
if doubles > singles {
n += singles * 4
quote = '\''
escapedQuote = singleQuoteEntityBytes
} else {
n += doubles * 4
quote = '"'
escapedQuote = doubleQuoteEntityBytes
}
if n > cap(*buf) {
*buf = make([]byte, 0, n) // maximum size, not actual size
}
t := (*buf)[:n] // maximum size, not actual size
t[0] = quote
j := 1
start := 0
for i, c := range b {
if c == quote {
j += copy(t[j:], b[start:i])
j += copy(t[j:], escapedQuote)
start = i + 1
}
}
j += copy(t[j:], b[start:])
t[j] = quote
return t[:j+1]
}
// EscapeCDATAVal returns the escaped text bytes.
func EscapeCDATAVal(buf *[]byte, b []byte) ([]byte, bool) {
n := 0
for _, c := range b {
if c == '<' || c == '&' {
if c == '<' {
n += 3 // <
} else {
n += 4 // &
}
if n > len("<![CDATA[]]>") {
return b, false
}
}
}
if len(b)+n > cap(*buf) {
*buf = make([]byte, 0, len(b)+n)
}
t := (*buf)[:len(b)+n]
j := 0
start := 0
for i, c := range b {
if c == '<' {
j += copy(t[j:], b[start:i])
j += copy(t[j:], ltEntityBytes)
start = i + 1
} else if c == '&' {
j += copy(t[j:], b[start:i])
j += copy(t[j:], ampEntityBytes)
start = i + 1
}
}
j += copy(t[j:], b[start:])
return t[:j], true
}
parse-2.5.21/xml/util_test.go 0000664 0000000 0000000 00000003123 14117006534 0016056 0 ustar 00root root 0000000 0000000 package xml
import (
"testing"
"github.com/tdewolff/test"
)
func TestEscapeAttrVal(t *testing.T) {
var attrValTests = []struct {
attrVal string
expected string
}{
{`xyz`, `"xyz"`},
{``, `""`},
{`x'z`, `"x'z"`},
{`x"z`, `'x"z'`},
{`a'b=""`, `'a'b=""'`},
{`'x'"'z'`, `"x'"'z"`},
{`"x"'"z"`, `'x"'"z'`},
{`a'b=""`, `'a'b=""'`},
}
var buf []byte
for _, tt := range attrValTests {
t.Run(tt.attrVal, func(t *testing.T) {
b := []byte(tt.attrVal)
if len(b) > 1 && (b[0] == '"' || b[0] == '\'') && b[0] == b[len(b)-1] {
b = b[1 : len(b)-1]
}
val := EscapeAttrVal(&buf, b)
test.String(t, string(val), tt.expected)
})
}
}
func TestEscapeCDATAVal(t *testing.T) {
var CDATAValTests = []struct {
CDATAVal string
expected string
}{
{"<![CDATA[<b>]]>", "<b>"},
{"<![CDATA[abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz]]>", "abcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyzabcdefghijklmnopqrstuvwxyz"},
{"<![CDATA[ <b> ]]>", " <b> "},
{"<![CDATA[<<<<<]]>", "<![CDATA[<<<<<]]>"},
{"<![CDATA[&]]>", "&"},
{"<![CDATA[&&&&]]>", "<![CDATA[&&&&]]>"},
{"<![CDATA[ a ]]>", " a "},
{"<![CDATA[]]>", ""},
{"<![CDATA[ a ]]> b ]]>", " a ]]&gt; b "},
}
var buf []byte
for _, tt := range CDATAValTests {
t.Run(tt.CDATAVal, func(t *testing.T) {
b := []byte(tt.CDATAVal[len("<![CDATA[") : len(tt.CDATAVal)-len("]]>")])
data, useText := EscapeCDATAVal(&buf, b)
text := string(data)
if !useText {
text = "<![CDATA[" + text + "]]>"
}
test.String(t, text, tt.expected)
})
}
}