pax_global_header00006660000000000000000000000064127314455300014516gustar00rootroot0000000000000052 comment=23fcbe4639fb4baee5dc53616958cc04c8b94026 js-tokens-2.0.0/000077500000000000000000000000001273144553000134325ustar00rootroot00000000000000js-tokens-2.0.0/.gitignore000066400000000000000000000000161273144553000154170ustar00rootroot00000000000000node_modules js-tokens-2.0.0/.travis.yml000066400000000000000000000000461273144553000155430ustar00rootroot00000000000000language: node_js node_js: - "node" js-tokens-2.0.0/LICENSE000066400000000000000000000021031273144553000144330ustar00rootroot00000000000000The MIT License (MIT) Copyright (c) 2014, 2015, 2016 Simon Lydell Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. js-tokens-2.0.0/changelog.md000066400000000000000000000047571273144553000157200ustar00rootroot00000000000000### Version 2.0.0 (2016-06-19) ### - Added: Support for ES2016. In other words, support for the `**` exponentiation operator. These are the breaking changes: - `'**'.match(jsTokens)` no longer returns `['*', '*']`, but `['**']`. - `'**='.match(jsTokens)` no longer returns `['*', '*=']`, but `['**=']`. ### Version 1.0.3 (2016-03-27) ### - Improved: Made the regex ever so slightly smaller. - Updated: The readme. ### Version 1.0.2 (2015-10-18) ### - Improved: Limited npm package contents for a smaller download. Thanks to @zertosh! ### Version 1.0.1 (2015-06-20) ### - Fixed: Declared an undeclared variable. ### Version 1.0.0 (2015-02-26) ### - Changed: Merged the 'operator' and 'punctuation' types into 'punctuator'. That type is now equivalent to the Punctuator token in the ECMAScript specification. (Backwards-incompatible change.) - Fixed: A `-` followed by a number is now correctly matched as a punctuator followed by a number. It used to be matched as just a number, but there is no such thing as negative number literals. (Possibly backwards-incompatible change.) ### Version 0.4.1 (2015-02-21) ### - Added: Support for the regex `u` flag. ### Version 0.4.0 (2015-02-21) ### - Improved: `jsTokens.matchToToken` performance. - Added: Support for octal and binary number literals. - Added: Support for template strings. ### Version 0.3.1 (2015-01-06) ### - Fixed: Support for unicode spaces. They used to be allowed in names (which is very confusing), and some unicode newlines were wrongly allowed in strings and regexes. ### Version 0.3.0 (2014-12-19) ### - Changed: The `jsTokens.names` array has been replaced with the `jsTokens.matchToToken` function. The capturing groups of `jsTokens` are no longer part of the public API; instead use said function. See this [gist] for an example. (Backwards-incompatible change.) - Changed: The empty string is now considered an “invalid” token, instead an “empty” token (its own group). (Backwards-incompatible change.) - Removed: component support. (Backwards-incompatible change.) [gist]: https://gist.github.com/lydell/be49dbf80c382c473004 ### Version 0.2.0 (2014-06-19) ### - Changed: Match ES6 function arrows (`=>`) as an operator, instead of its own category (“functionArrow”), for simplicity. (Backwards-incompatible change.) - Added: ES6 splats (`...`) are now matched as an operator (instead of three punctuations). (Backwards-incompatible change.) ### Version 0.1.0 (2014-03-08) ### - Initial release. js-tokens-2.0.0/esprima-compare.js000066400000000000000000000045741273144553000170660ustar00rootroot00000000000000// Copyright 2015 Simon Lydell // X11 (“MIT”) Licensed. (See LICENSE.) var fs = require("fs") var esprima = require("esprima") var jsTokens = require("./") var typeMap = { Boolean: "name", Identifier: "name", Keyword: "name", Null: "name", Numeric: "number", Punctuator: "punctuator", RegularExpression: "regex", String: "string" } function getEsprimaTokens(code) { var tokens = esprima.tokenize(code, {loc: true}) tokens.forEach(function(token) { token.type = typeMap[token.type] }) return tokens } function jsTokensTokenize(string) { jsTokens.lastIndex = 0 if (string === "") return [] var tokens = [] var match while (match = jsTokens.exec(string)) { tokens.push(jsTokens.matchToToken(match)) } return tokens } var exclusionMap = { comment: true, whitespace: true } function getJsTokensTokens(code) { return jsTokensTokenize(code) .filter(function(token) { return !exclusionMap.hasOwnProperty(token.type) }) } function compare(file) { var code = fs.readFileSync(require.resolve(file)).toString() var esprimaTokens = getEsprimaTokens(code) var jsTokensTokens = getJsTokensTokens(code) var length = Math.min(esprimaTokens.length, jsTokensTokens.length) for (var index = 0; index < length; index++) { var esprimaToken = esprimaTokens[index] var jsTokensToken = jsTokensTokens[index] if ( esprimaToken.type !== jsTokensToken.type || esprimaToken.value !== jsTokensToken.value ) { var loc = esprimaToken.loc.start console.error( file + ":" + loc.line + ":" + (loc.column + 1) + ": " + "(token #" + (index + 1) + ")\n" + " esprima: '" + esprimaToken.type + "': " + esprimaToken.value + "\n" + " jsTokens: '" + jsTokensToken.type + "': " + jsTokensToken.value ) return false } } if (esprimaTokens.length !== jsTokensTokens.length) { console.error( file + ': Number of tokens mismatch.\n' + " esprima: " + (esprimaTokens.length + 1) + "\n" + " jsTokens: " + (jsTokensTokens.length + 1) ) return false } return true } var results = process.argv.slice(2).map(compare) if (results.every(Boolean)) { console.log( "Comparison succeeded: esprima and jsTokens produced the same tokens!" ) } else { console.error("Comparison failed.") } js-tokens-2.0.0/generate-index.js000066400000000000000000000004641273144553000166730ustar00rootroot00000000000000// Copyright 2014 Simon Lydell // X11 (“MIT”) Licensed. (See LICENSE.) var fs = require("fs") require("coffee-script/register") var regex = require("./regex.coffee") var code = fs.readFileSync("index.js").toString() code = code.replace(/\/.+\/.+/, regex.toString()) fs.writeFileSync("index.js", code) js-tokens-2.0.0/index.js000066400000000000000000000025351273144553000151040ustar00rootroot00000000000000// Copyright 2014, 2015, 2016 Simon Lydell // X11 (“MIT”) Licensed. (See LICENSE.) // This regex comes from regex.coffee, and is inserted here by generate-index.js // (run `npm run build`). module.exports = /((['"])(?:(?!\2|\\).|\\(?:\r\n|[\s\S]))*(\2)?|`(?:[^`\\$]|\\[\s\S]|\$(?!\{)|\$\{(?:[^{}]|\{[^}]*\}?)*\}?)*(`)?)|(\/\/.*)|(\/\*(?:[^*]|\*(?!\/))*(\*\/)?)|(\/(?!\*)(?:\[(?:(?![\]\\]).|\\.)*\]|(?![\/\]\\]).|\\.)+\/(?:(?!\s*(?:\b|[\u0080-\uFFFF$\\'"~({]|[+\-!](?!=)|\.?\d))|[gmiyu]{1,5}\b(?![\u0080-\uFFFF$\\]|\s*(?:[+\-*%&|^<>!=?({]|\/(?![\/*])))))|(0[xX][\da-fA-F]+|0[oO][0-7]+|0[bB][01]+|(?:\d*\.\d+|\d+\.?)(?:[eE][+-]?\d+)?)|((?!\d)(?:(?!\s)[$\w\u0080-\uFFFF]|\\u[\da-fA-F]{4}|\\u\{[\da-fA-F]{1,6}\})+)|(--|\+\+|&&|\|\||=>|\.{3}|(?:[+\-\/%&|^]|\*{1,2}|<{1,2}|>{1,3}|!=?|={1,2})=?|[?~.,:;[\](){}])|(\s+)|(^$|[\s\S])/g module.exports.matchToToken = function(match) { var token = {type: "invalid", value: match[0]} if (match[ 1]) token.type = "string" , token.closed = !!(match[3] || match[4]) else if (match[ 5]) token.type = "comment" else if (match[ 6]) token.type = "comment", token.closed = !!match[7] else if (match[ 8]) token.type = "regex" else if (match[ 9]) token.type = "number" else if (match[10]) token.type = "name" else if (match[11]) token.type = "punctuator" else if (match[12]) token.type = "whitespace" return token } js-tokens-2.0.0/package.json000066400000000000000000000012171273144553000157210ustar00rootroot00000000000000{ "name": "js-tokens", "version": "2.0.0", "author": "Simon Lydell", "license": "MIT", "description": "A regex that tokenizes JavaScript.", "keywords": [ "JavaScript", "js", "token", "tokenize", "regex" ], "files": [ "index.js" ], "repository": "lydell/js-tokens", "scripts": { "test": "mocha --ui tdd", "esprima-compare": "node esprima-compare ./index.js everything.js/es5.js", "build": "node generate-index.js", "dev": "npm run build && npm test" }, "devDependencies": { "coffee-script": "~1.10.0", "esprima": "^2.7.2", "everything.js": "^1.0.3", "mocha": "^2.5.3" } } js-tokens-2.0.0/readme.md000066400000000000000000000145351273144553000152210ustar00rootroot00000000000000Overview [![Build Status](https://travis-ci.org/lydell/js-tokens.png?branch=master)](https://travis-ci.org/lydell/js-tokens) ======== A regex that tokenizes JavaScript. ```js var jsTokens = require("js-tokens") var jsString = "var foo=opts.foo;\n..." jsString.match(jsTokens) // ["var", " ", "foo", "=", "opts", ".", "foo", ";", "\n", ...] ``` Installation ============ `npm install js-tokens` ```js var jsTokens = require("js-tokens") ``` Usage ===== ### `jsTokens` ### A regex with the `g` flag that matches JavaScript tokens. The regex _always_ matches, even invalid JavaScript and the empty string. The next match is always directly after the previous. ### `var token = jsTokens.matchToToken(match)` ### Takes a `match` returned by `jsTokens.exec(string)`, and returns a `{type: String, value: String}` object. The following types are available: - string - comment - regex - number - name - punctuator - whitespace - invalid Multi-line comments and strings also have a `closed` property indicating if the token was closed or not (see below). Comments and strings both come in several flavors. To distinguish them, check if the token starts with `//`, `/*`, `'`, `"` or `` ` ``. Names are ECMAScript IdentifierNames, that is, including both identifiers and keywords. You may use [is-keyword-js] to tell them apart. Whitespace includes both line terminators and other whitespace. For example usage, please see this [gist]. [is-keyword-js]: https://github.com/crissdev/is-keyword-js [gist]: https://gist.github.com/lydell/be49dbf80c382c473004 ECMAScript support ================== The intention is to always support the latest stable ECMAScript version. If adding support for a newer version requires changes, a new version with a major verion bump will be released. Currently, [ECMAScript 2016] is supported. [ECMAScript 2016]: http://www.ecma-international.org/ecma-262/7.0/index.html Invalid code handling ===================== Unterminated strings are still matched as strings. JavaScript strings cannot contain (unescaped) newlines, so unterminated strings simply end at the end of the line. Unterminated template strings can contain unescaped newlines, though, so they go on to the end of input. Unterminated multi-line comments are also still matched as comments. They simply go on to the end of the input. Unterminated regex literals are likely matched as division and whatever is inside the regex. Invalid ASCII characters have their own capturing group. Invalid non-ASCII characters are treated as names, to simplify the matching of names (except unicode spaces which are treated as whitespace). Regex literals may contain invalid regex syntax. They are still matched as regex literals. They may also contain repeated regex flags, to keep the regex simple. Strings may contain invalid escape sequences. Limitations =========== Tokenizing JavaScript using regexes—in fact, _one single regex_—won’t be perfect. But that’s not the point either. You may compare jsTokens with [esprima] by using `esprima-compare.js`. See `npm run esprima-compare`! [esprima]: http://esprima.org/ ### Template string interpolation ### Template strings are matched as single tokens, from the starting `` ` `` to the ending `` ` ``, including interpolations (whose tokens are not matched individually). Matching template string interpolations requires recursive balancing of `{` and `}`—something that JavaScript regexes cannot do. Only one level of nesting is supported. ### Division and regex literals collision ### Consider this example: ```js var g = 9.82 var number = bar / 2/g var regex = / 2/g ``` A human can easily understand that in the `number` line we’re dealing with division, and in the `regex` line we’re dealing with a regex literal. How come? Because humans can look at the whole code to put the `/` characters in context. A JavaScript regex cannot. It only sees forwards. When the `jsTokens` regex scans throught the above, it will see the following at the end of both the `number` and `regex` rows: ```js / 2/g ``` It is then impossible to know if that is a regex literal, or part of an expression dealing with division. Here is a similar case: ```js foo /= 2/g foo(/= 2/g) ``` The first line divides the `foo` variable with `2/g`. The second line calls the `foo` function with the regex literal `/= 2/g`. Again, since `jsTokens` only sees forwards, it cannot tell the two cases apart. There are some cases where we _can_ tell division and regex literals apart, though. First off, we have the simple cases where there’s only one slash in the line: ```js var foo = 2/g foo /= 2 ``` Regex literals cannot contain newlines, so the above cases are correctly identified as division. Things are only problematic when there are more than one non-comment slash in a single line. Secondly, not every character is a valid regex flag. ```js var number = bar / 2/e ``` The above example is also correctly identified as division, because `e` is not a valid regex flag. I initially wanted to future-proof by allowing `[a-zA-Z]*` (any letter) as flags, but it is not worth it since it increases the amount of ambigous cases. So only the standard `g`, `m`, `i`, `y` and `u` flags are allowed. This means that the above example will be identified as division as long as you don’t rename the `e` variable to some permutation of `gmiyu` 1 to 5 characters long. Lastly, we can look _forward_ for information. - If the token following what looks like a regex literal is not valid after a regex literal, but is valid in a division expression, then the regex literal is treated as division instead. For example, a flagless regex cannot be followed by a string, number or name, but all of those three can be the denominator of a division. - Generally, if what looks like a regex literal is followed by an operator, the regex literal is treated as division instead. This is because regexes are seldomly used with operators (such as `+`, `*`, `&&` and `==`), but division could likely be part of such an expression. Please consult the regex source and the test cases for precise information on when regex or division is matched (should you need to know). In short, you could sum it up as: If the end of a statement looks like a regex literal (even if it isn’t), it will be treated as one. Otherwise it should work as expected (if you write sane code). License ======= [The X11 (“MIT”) License](LICENSE). js-tokens-2.0.0/regex.coffee000066400000000000000000000043051273144553000157170ustar00rootroot00000000000000# Copyright 2014, 2015, 2016 Simon Lydell # X11 (“MIT”) Licensed. (See LICENSE.) # # Don’t worry, you don’t need to know CoffeeScript. It is only used for its # readable regex syntax. Everything else is done in JavaScript in index.js. module.exports = /// ( # ([ ' " ]) (?: (?! \2 | \\ ). | \\(?: \r\n | [\s\S] ) )* (\2)? | ` (?: [^ ` \\ $ ] | \\[\s\S] | \$(?!\{) | \$\{ (?: [^{}] | \{ [^}]* \}? )* \}? )* (`)? ) | ( # //.* ) | ( # /\* (?: [^*] | \*(?!/) )* ( \*/ )? ) | ( # /(?!\*) (?: \[ (?: (?![ \] \\ ]). | \\. )* \] | (?![ / \] \\ ]). | \\. )+ / (?: (?! \s* (?: \b | [ \u0080-\uFFFF $ \\ ' " ~ ( { ] | [ + \- ! ](?!=) | \.?\d ) ) | [ g m i y u ]{1,5} \b (?! [ \u0080-\uFFFF $ \\ ] | \s* (?: [ + \- * % & | ^ < > ! = ? ( { ] | /(?! [ / * ] ) ) ) ) ) | ( # 0[xX][ \d a-f A-F ]+ | 0[oO][0-7]+ | 0[bB][01]+ | (?: \d*\.\d+ | \d+\.? # Support one trailing dot for integers only. ) (?: [eE][+-]?\d+ )? ) | ( # # See . (?!\d) (?: (?!\s)[ $ \w \u0080-\uFFFF ] | \\u[ \d a-f A-F ]{4} | \\u\{[ \d a-f A-F ]{1,6}\} )+ ) | ( # -- | \+\+ | && | \|\| | => | \.{3} | (?: [ + \- / % & | ^ ] | \*{1,2} | <{1,2} | >{1,3} | !=? | ={1,2} )=? | [ ? ~ . , : ; [ \] ( ) { } ] ) | ( # \s+ ) | ( # ^$ # Empty. | [\s\S] # Catch-all rule for anything not matched by the above. ) ///g js-tokens-2.0.0/test/000077500000000000000000000000001273144553000144115ustar00rootroot00000000000000js-tokens-2.0.0/test/fixtures/000077500000000000000000000000001273144553000162625ustar00rootroot00000000000000js-tokens-2.0.0/test/fixtures/base64.js000066400000000000000000000043021273144553000177030ustar00rootroot00000000000000/* * https://github.com/davidchambers/Base64.js */ ;(function () { var object = typeof exports != 'undefined' ? exports : this; // #8: web workers var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='; function InvalidCharacterError(message) { this.message = message; } InvalidCharacterError.prototype = new Error; InvalidCharacterError.prototype.name = 'InvalidCharacterError'; // encoder // [https://gist.github.com/999166] by [https://github.com/nignag] object.btoa || ( object.btoa = function (input) { for ( // initialize result and counter var block, charCode, idx = 0, map = chars, output = ''; // if the next input index does not exist: // change the mapping table to "=" // check if d has no fractional digits input.charAt(idx | 0) || (map = '=', idx % 1); // "8 - idx % 1 * 8" generates the sequence 2, 4, 6, 8 output += map.charAt(63 & block >> 8 - idx % 1 * 8) ) { charCode = input.charCodeAt(idx += 3/4); if (charCode > 0xFF) { throw new InvalidCharacterError("'btoa' failed: The string to be encoded contains characters outside of the Latin1 range."); } block = block << 8 | charCode; } return output; }); // decoder // [https://gist.github.com/1020396] by [https://github.com/atk] object.atob || ( object.atob = function (input) { input = input.replace(/=+$/, '') if (input.length % 4 == 1) { throw new InvalidCharacterError("'atob' failed: The string to be decoded is not correctly encoded."); } for ( // initialize result and counters var bc = 0, bs, buffer, idx = 0, output = ''; // get next character buffer = input.charAt(idx++); // character found in table? initialize bit storage and add its ascii value; ~buffer && (bs = bc % 4 ? bs * 64 + buffer : buffer, // and if not first of each 4 characters, // convert the first 8 bits to one ascii character bc++ % 4) ? output += String.fromCharCode(255 & bs >> (-2 * bc & 6)) : 0 ) { // try to find character in table (0-63, not found => -1) buffer = chars.indexOf(buffer); } return output; }); }()); js-tokens-2.0.0/test/fixtures/base64.json000066400000000000000000000075631273144553000202540ustar00rootroot00000000000000[ "/*\n * https://github.com/davidchambers/Base64.js\n */","\n", ";","(","function"," ","(",")"," ","{","\n\n ", "var"," ","object"," ","="," ","typeof"," ","exports"," ","!="," ","'undefined'"," ","?"," ","exports"," ",":"," ","this",";"," ","// #8: web workers","\n ", "var"," ","chars"," ","="," ","'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/='",";","\n\n ", "function"," ","InvalidCharacterError","(","message",")"," ","{","\n ", "this",".","message"," ","="," ","message",";","\n ", "}","\n ", "InvalidCharacterError",".","prototype"," ","="," ","new"," ","Error",";","\n ", "InvalidCharacterError",".","prototype",".","name"," ","="," ","'InvalidCharacterError'",";","\n\n ", "// encoder","\n ", "// [https://gist.github.com/999166] by [https://github.com/nignag]","\n ", "object",".","btoa"," ","||"," ","(","\n ", "object",".","btoa"," ","="," ","function"," ","(","input",")"," ","{","\n ", "for"," ","(","\n ", "// initialize result and counter","\n ", "var"," ","block",","," ","charCode",","," ","idx"," ","="," ","0",","," ","map"," ","="," ","chars",","," ","output"," ","="," ","''",";","\n ", "// if the next input index does not exist:","\n ", "// change the mapping table to \"=\"","\n ", "// check if d has no fractional digits","\n ", "input",".","charAt","(","idx"," ","|"," ","0",")"," ","||"," ","(","map"," ","="," ","'='",","," ","idx"," ","%"," ","1",")",";","\n ", "// \"8 - idx % 1 * 8\" generates the sequence 2, 4, 6, 8","\n ", "output"," ","+="," ","map",".","charAt","(","63"," ","&"," ","block"," ",">>"," ","8"," ","-"," ","idx"," ","%"," ","1"," ","*"," ","8",")","\n ", ")"," ","{","\n ", "charCode"," ","="," ","input",".","charCodeAt","(","idx"," ","+="," ","3","/","4",")",";","\n ", "if"," ","(","charCode"," ",">"," ","0xFF",")"," ","{","\n ", "throw"," ","new"," ","InvalidCharacterError","(","\"'btoa' failed: The string to be encoded contains characters outside of the Latin1 range.\"",")",";","\n ", "}","\n ", "block"," ","="," ","block"," ","<<"," ","8"," ","|"," ","charCode",";","\n ", "}","\n ", "return"," ","output",";","\n ", "}",")",";","\n\n ", "// decoder","\n ", "// [https://gist.github.com/1020396] by [https://github.com/atk]","\n ", "object",".","atob"," ","||"," ","(","\n ", "object",".","atob"," ","="," ","function"," ","(","input",")"," ","{","\n ", "input"," ","="," ","input",".","replace","(","/=+$/",","," ","''",")","\n ", "if"," ","(","input",".","length"," ","%"," ","4"," ","=="," ","1",")"," ","{","\n ", "throw"," ","new"," ","InvalidCharacterError","(","\"'atob' failed: The string to be decoded is not correctly encoded.\"",")",";","\n ", "}","\n ", "for"," ","(","\n ", "// initialize result and counters","\n ", "var"," ","bc"," ","="," ","0",","," ","bs",","," ","buffer",","," ","idx"," ","="," ","0",","," ","output"," ","="," ","''",";","\n ", "// get next character","\n ", "buffer"," ","="," ","input",".","charAt","(","idx","++",")",";","\n ", "// character found in table? initialize bit storage and add its ascii value;","\n ", "~","buffer"," ","&&"," ","(","bs"," ","="," ","bc"," ","%"," ","4"," ","?"," ","bs"," ","*"," ","64"," ","+"," ","buffer"," ",":"," ","buffer",",","\n ", "// and if not first of each 4 characters,","\n ", "// convert the first 8 bits to one ascii character","\n ", "bc","++"," ","%"," ","4",")"," ","?"," ","output"," ","+="," ","String",".","fromCharCode","(","255"," ","&"," ","bs"," ",">>"," ","(","-","2"," ","*"," ","bc"," ","&"," ","6",")",")"," ",":"," ","0","\n ", ")"," ","{","\n ", "// try to find character in table (0-63, not found => -1)","\n ", "buffer"," ","="," ","chars",".","indexOf","(","buffer",")",";","\n ", "}","\n ", "return"," ","output",";","\n ", "}",")",";","\n\n", "}","(",")",")",";","\n" ] js-tokens-2.0.0/test/fixtures/division.js000066400000000000000000000007331273144553000204470ustar00rootroot00000000000000foo = 1/a/2 foo = 1/a/ 2 foo = 1/a/(b+1) foo = 1/a/~bar.indexOf(baz) foo = 1/a/+str foo = 1/a/-1 foo = 1/a/-bar foo = 1/a/--bar foo = 1/a/ --bar foo = 1/a/ ++bar foo = 1/a/g+1 foo = 1/a/g - 1 foo = 1/a/g*1 foo = 1/a/g/1 foo = 1/a/g /1 if (1/a/g && foo) {} if (1/a/g && foo) {} if (1/a/g || foo) {} if (1/a/g === 3) {} foo = 1/a/g ? true : false nan = 1/a/'' nan = 1/a/ "" foo = 1/a/goo foo = 1/a/iff foo = 1/a/igor foo = 1/a/moo foo = 1/a/yüm foo = 1/a/imgyp js-tokens-2.0.0/test/fixtures/division.json000066400000000000000000000030751273144553000210060ustar00rootroot00000000000000[ "foo"," ","="," ","1","/","a","/","2","\n", "foo"," ","="," ","1","/","a","/","\n ","2","\n", "foo"," ","="," ","1","/","a","/","(","b","+","1",")","\n", "foo"," ","="," ","1","/","a","/","~","bar",".","indexOf","(","baz",")","\n", "foo"," ","="," ","1","/","a","/","+","str","\n", "foo"," ","="," ","1","/","a","/","-","1","\n", "foo"," ","="," ","1","/","a","/","-","bar","\n", "foo"," ","="," ","1","/","a","/","--","bar","\n", "foo"," ","="," ","1","/","a","/","\n ","--","bar","\n", "foo"," ","="," ","1","/","a","/"," ","++","bar","\n\n", "foo"," ","="," ","1","/","a","/","g","+","1","\n", "foo"," ","="," ","1","/","a","/","g"," ","-"," ","1","\n", "foo"," ","="," ","1","/","a","/","g","*","1","\n", "foo"," ","="," ","1","/","a","/","g","/","1","\n", "foo"," ","="," ","1","/","a","/","g","\n ","/","1","\n\n", "if"," ","(","1","/","a","/","g"," ","&&"," ","foo",")"," ","{","}","\n", "if"," ","(","1","/","a","/","g","\n ","&&"," ","foo",")"," ","{","}","\n", "if"," ","(","1","/","a","/","g"," ","||"," ","foo",")"," ","{","}","\n", "if"," ","(","1","/","a","/","g"," ","==="," ","3",")"," ","{","}","\n\n", "foo"," ","="," ","1","/","a","/","g"," ","?"," ","true"," ",":"," ","false","\n\n", "nan"," ","="," ","1","/","a","/","''","\n", "nan"," ","="," ","1","/","a","/"," ","\"\"","\n\n", "foo"," ","="," ","1","/","a","/","goo","\n", "foo"," ","="," ","1","/","a","/","iff","\n", "foo"," ","="," ","1","/","a","/","igor","\n", "foo"," ","="," ","1","/","a","/","moo","\n", "foo"," ","="," ","1","/","a","/","yüm","\n", "foo"," ","="," ","1","/","a","/","imgyp","\n" ] js-tokens-2.0.0/test/fixtures/errors.js000066400000000000000000000004421273144553000201340ustar00rootroot00000000000000var multiLineString = '\ While I was writing along\ I suddenly forgot to add a backslash @ the end of a line' var unterminatedRegex = /3?2:1 # Oops, forgot that this isn’t coffee-script. /* Let’s try a valid JavaScript comment instead var string='Pity I forgot to close it, though' js-tokens-2.0.0/test/fixtures/errors.json000066400000000000000000000007541273144553000204770ustar00rootroot00000000000000[ "var"," ","multiLineString"," ","="," ","'\\\nWhile I was writing along\\\nI suddenly forgot to","\n", "add"," ","a"," ","backslash"," ","@"," ","the"," ","end"," ","of"," ","a"," ","line","'","\n\n", "var"," ","unterminatedRegex"," ","="," ","/","3","?","2",":","1","\n\n", "#"," ","Oops",","," ","forgot"," ","that"," ","this"," ","isn’t"," ","coffee","-","script",".","\n\n", "/* Let’s try a valid JavaScript comment instead\n\nvar string='Pity I forgot to close it, though'\n" ] js-tokens-2.0.0/test/fixtures/regex.js000066400000000000000000000011761273144553000177370ustar00rootroot00000000000000foo(/a/) foo(/a/g) foo( /a/ ) foo( /a/g ) foo(/a/, bar) foo(/a/g, bar) foo(/a/, /a/g, /b/g, /b/) arr = [/a/] arr = [/a/g] arr = [ /a/ ] arr = [ /a/g ] obj = {re: /a/} obj = {re: /a/g} obj = { re: /a/ } obj = { re: /a/g } re = foo ? /a/ : RegExp(bar) re = foo ? /a/g : RegExp(bar) /a/.exec(foo) /a/g.exec(foo) foo = (1/2) + /a/.exec(bar)[0] foo = (1/2) + /a/g.exec(bar)[0] re = /a/// comment re = /a/g// comment re = /a//* comment */ re = /a/g/* comment */ re = /a/ // comment re = /a/g // comment re = /a/ /* comment */ re = /a/g /* comment */ silly = /a/ ? true : false if (/a/ == "/a/") {} if (/a/ && foo) {} if (/a/ || foo) {} js-tokens-2.0.0/test/fixtures/regex.json000066400000000000000000000033321273144553000202700ustar00rootroot00000000000000[ "foo","(","/a/",")","\n", "foo","(","/a/g",")","\n", "foo","("," ","/a/"," ",")","\n", "foo","("," ","/a/g"," ",")","\n", "foo","(","/a/",","," ","bar",")","\n", "foo","(","/a/g",","," ","bar",")","\n", "foo","(","/a/",","," ","/a/g",","," ","/b/g",","," ","/b/",")","\n\n", "arr"," ","="," ","[","/a/","]","\n", "arr"," ","="," ","[","/a/g","]","\n", "arr"," ","="," ","["," ","/a/"," ","]","\n", "arr"," ","="," ","["," ","/a/g"," ","]","\n\n", "obj"," ","="," ","{","re",":"," ","/a/","}","\n", "obj"," ","="," ","{","re",":"," ","/a/g","}","\n", "obj"," ","="," ","{"," ","re",":"," ","/a/"," ","}","\n", "obj"," ","="," ","{"," ","re",":"," ","/a/g"," ","}","\n\n", "re"," ","="," ","foo"," ","?"," ","/a/"," ",":"," ","RegExp","(","bar",")","\n", "re"," ","="," ","foo"," ","?"," ","/a/g"," ",":"," ","RegExp","(","bar",")","\n\n", "/a/",".","exec","(","foo",")","\n", "/a/g",".","exec","(","foo",")","\n\n", "foo"," ","="," ","(","1","/","2",")"," ","+"," ","/a/",".","exec","(","bar",")","[","0","]","\n", "foo"," ","="," ","(","1","/","2",")"," ","+"," ","/a/g",".","exec","(","bar",")","[","0","]","\n\n", "re"," ","="," ","/a/","// comment","\n", "re"," ","="," ","/a/g","// comment","\n", "re"," ","="," ","/a/","/* comment */","\n", "re"," ","="," ","/a/g","/* comment */","\n\n", "re"," ","="," ","/a/"," ","// comment","\n", "re"," ","="," ","/a/g"," ","// comment","\n", "re"," ","="," ","/a/"," ","/* comment */","\n", "re"," ","="," ","/a/g"," ","/* comment */","\n\n", "silly"," ","="," ","/a/"," ","?"," ","true"," ",":"," ","false","\n", "if"," ","(","/a/"," ","=="," ","\"/a/\"",")"," ","{","}","\n", "if"," ","(","/a/"," ","&&"," ","foo",")"," ","{","}","\n", "if"," ","(","/a/"," ","||"," ","foo",")"," ","{","}","\n" ] js-tokens-2.0.0/test/index.js000066400000000000000000000465201273144553000160650ustar00rootroot00000000000000// Copyright 2014, 2015, 2016 Simon Lydell // X11 (“MIT”) Licensed. (See LICENSE.) var fs = require("fs") var util = require("util") var assert = require("assert") var jsTokens = require("../") suite("jsTokens", function() { test("is a regex", function() { assert(util.isRegExp(jsTokens)) }) }) suite("jsTokens.matchToToken", function() { test("is a function", function() { assert.equal(typeof jsTokens.matchToToken, "function") }) }) suite("tokens", function() { function token(name, fn) { suite(name, fn.bind(null, matchHelper.bind(null, name))) } function matchHelper(type, string, expected, extra) { extra = extra || {} if (typeof expected === "object") { extra = expected expected = undefined } jsTokens.lastIndex = 0 var token = jsTokens.matchToToken(jsTokens.exec(string)) test(string, function() { if (expected === false) { assert.notEqual(token.type, type) } else { assert.equal(token.type, type) assert.equal( token.value, (typeof expected === "string" ? expected : string) ) if ("closed" in extra) { assert.equal(token.closed, extra.closed) } else if (type === "string") { assert.equal(token.closed, true) } } }) } token("whitespace", function(match) { match(" ") match(" ") match(" a", " ") match("\t") match("\t\t\t") match("\ta", "\t") match("\n") match("\n\n\n") match("\na", "\n") match("\r") match("\r\r\r") match("\ra", "\r") match(" \t\n\r \r\n") match(" \t\n\r \r\n-1", " \t\n\r \r\n") match("\f") match("\v") match("\u00a0") match("\u1680") match("\u180e") match("\u2000") match("\u2001") match("\u2002") match("\u2003") match("\u2004") match("\u2005") match("\u2006") match("\u2007") match("\u2008") match("\u2009") match("\u200a") match("\u2028") match("\u2029") match("\u202f") match("\u205f") match("\u3000") }) token("comment", function(match) { match("//") match("//comment") match("// comment") match("//comment ") match("///") match("//**//") match("//comment\n", "//comment") match("//comment\r", "//comment") match("//comment\u2028", "//comment") match("//comment\u2029", "//comment") match("//comment\r\n", "//comment") match("//comment \n", "//comment ") match("//comment\t\n", "//comment\t") match("/**/", {closed: true}) match("/*comment*/", {closed: true}) match("/* comment */", {closed: true}) match("/***/", {closed: true}) match("/*/*/", {closed: true}) match("/*\n\r\u2028\u2029 \r\n*/", {closed: true}) match("/*", {closed: false}) match("/*/", {closed: false}) match("/*unclosed", {closed: false}) match("/*unclosed\nnew Line(this == code ? true : false)", {closed: false}) }) token("string", function(match) { match("''") match('""') match("``") match("'string'") match('"string"') match("`string`") match("'\\''") match('"\\""') match("`\\``") match("'\\\\''", "'\\\\'") match('"\\\\""', '"\\\\"') match("`\\\\``", "`\\\\`") match("'\\\\\\''") match('"\\\\\\""') match("`\\\\\\``") match("'\\u05aF'") match('"\\u05aF"') match("`\\u05aF`") match("'invalid escape sequence is OK: \\u'") match('"invalid escape sequence is OK: \\u"') match("`invalid escape sequence is OK: \\u`") match("'\\\n'") match('"\\\n"') match("`\\\n`") match("'\\\r'") match('"\\\r"') match("`\\\r`") match("'\\\u2028'") match('"\\\u2028"') match("`\\\u2028`") match("'\\\u2029'") match('"\\\u2029"') match("`\\\u2029`") match("'\\\r\n'") match('"\\\r\n"') match("`\\\r\n`") match("'string'code'another string'", "'string'") match('"string"code"another string"', '"string"') match("`string`code`another string`", "`string`") match("'\"'") match("'`'") match('"\'"') match('"`"') match("`'`") match('`"`') match("'", {closed: false}) match('"', {closed: false}) match("`", {closed: false}) match("'unclosed", {closed: false}) match('"unclosed', {closed: false}) match("`unclosed", {closed: false}) match("'\n", "'", {closed: false}) match('"\n', '"', {closed: false}) match("`\n", {closed: false}) match("'\r", "'", {closed: false}) match('"\r', '"', {closed: false}) match("`\r", {closed: false}) match("'\u2028", "'", {closed: false}) match('"\u2028', '"', {closed: false}) match("`\u2028", {closed: false}) match("'\u2029", "'", {closed: false}) match('"\u2029', '"', {closed: false}) match("`\u2029", {closed: false}) match("'\r\n", "'", {closed: false}) match('"\r\n', '"', {closed: false}) match("`\r\n", {closed: false}) match("'\\\n", {closed: false}) match('"\\\n', {closed: false}) match("`\\\n", {closed: false}) match("'\\\r", {closed: false}) match('"\\\r', {closed: false}) match("`\\\r", {closed: false}) match("'\\\u2028", {closed: false}) match('"\\\u2028', {closed: false}) match("`\\\u2028", {closed: false}) match("'\\\u2029", {closed: false}) match('"\\\u2029', {closed: false}) match("`\\\u2029", {closed: false}) match("'\\\r\n", {closed: false}) match('"\\\r\n', {closed: false}) match("`\\\r\n", {closed: false}) match("'${}'") match('"${}"') match("`${}`") match("'${a}'") match('"${a}"') match("`${a}`") match("'a${b}c'") match('"a${b}c"') match("`a${b}c`") match("'${'a'}'", "'${'") match('"${"a"}"', '"${"') match("`${`a`}`") match("`${`${a}`}`") match("`${fn({a: b})}`") match("`${fn({a: '{'})}`") match("`a${}${a}${ `${b\r}` + `${`c`}` } d $${\n(x=>{return x*2})(4)}$`") match("`\\${{{}}}a`") match("`a ${b c`.length", {closed: false}) match("`a ${`b${c`} d`.length", {closed: false}) match("`a ${ {c:d } e`.length", {closed: false}) }) token("regex", function(match) { match("//", false) match("/a/") match("/\\//") match("/\\\\//", "/\\\\/") match("/\\\\\\//") match("/[/]/") match("/[\\]]/") match("/[\\]/]/") match("/[\\\\]/]/", "/[\\\\]/") match("/[\\\\\\]/]/") match(/\\u05aF/) match("/invalid escape sequence is OK: \\u/") match("/?foo/") match("/*foo/", false) match("/a/g") match("/a/m") match("/a/i") match("/a/y") match("/a/u") match("/a/gmiyu") match("/a/myg") match("/a/e", false) match("/a/invalidFlags", false) match("/a/f00", false) match("/\n/", false) match("/\r/", false) match("/\u2028/", false) match("/\u2029/", false) match("/\r\n/", false) match("/\\\n/", false) match("/\\\r/", false) match("/\\\u2028/", false) match("/\\\u2029/", false) match("/\\\r\n/", false) match("/[\n]/", false) match("/[\r]/", false) match("/[\u2028]/", false) match("/[\u2029]/", false) match("/[\r\n]/", false) match("/[\\\n]/", false) match("/[\\\r]/", false) match("/[\\\u2028]/", false) match("/[\\\u2029]/", false) match("/[\\\r\n]/", false) match("/a/", "/a/") match("/a/g", "/a/g") match("/a/;", "/a/") match("/a/g;", "/a/g") match("/a/ ;", "/a/") match("/a/g ;", "/a/g") match("/a/, b", "/a/") match("/a/g, b", "/a/g") match("/a/ , b", "/a/") match("/a/g , b", "/a/g") match("/a/.exec(b)", "/a/") match("/a/g.exec(b)", "/a/g") match("/a/ .exec(b)", "/a/") match("/a/g .exec(b)", "/a/g") match("/a/['exec'](b)", "/a/") match("/a/g['exec'](b)", "/a/g") match("/a/ ['exec'](b)", "/a/") match("/a/g ['exec'](b)", "/a/g") match("/a/]", "/a/") match("/a/g]", "/a/g") match("/a/ ]", "/a/") match("/a/g ]", "/a/g") match("/a/)", "/a/") match("/a/g)", "/a/g") match("/a/ )", "/a/") match("/a/g )", "/a/g") match("/a/}", "/a/") match("/a/g}", "/a/g") match("/a/ }", "/a/") match("/a/g }", "/a/g") match("/a/+=b", "/a/") match("/a/ +=b", "/a/") match("/a/-=b", "/a/") match("/a/ -=b", "/a/") match("/a/*b", "/a/") match("/a/ *b", "/a/") match("/a/ *=b", "/a/") match("/a//b", "/a/") match("/a/ /b", "/a/") match("/a/ /=b", "/a/") match("/a/%b", "/a/") match("/a/ %b", "/a/") match("/a/%=b", "/a/") match("/a/ %=b", "/a/") match("/a/&b", "/a/") match("/a/ &b", "/a/") match("/a/&=b", "/a/") match("/a/ &=b", "/a/") match("/a/&&b", "/a/") match("/a/ &&b", "/a/") match("/a/|b", "/a/") match("/a/ |b", "/a/") match("/a/|=b", "/a/") match("/a/ |=b", "/a/") match("/a/||b", "/a/") match("/a/ ||b", "/a/") match("/a/^b", "/a/") match("/a/ ^b", "/a/") match("/a/^=b", "/a/") match("/a/ ^=b", "/a/") match("/a/b", "/a/") match("/a/ >b", "/a/") match("/a/>=b", "/a/") match("/a/ >=b", "/a/") match("/a/>>b", "/a/") match("/a/ >>b", "/a/") match("/a/>>=b", "/a/") match("/a/ >>=b", "/a/") match("/a/>>>b", "/a/") match("/a/ >>>=b", "/a/") match("/a/>>>=b", "/a/") match("/a/ >>>b", "/a/") match("/a/!=b", "/a/") match("/a/ !=b", "/a/") match("/a/!==b", "/a/") match("/a/ !==b", "/a/") match("/a/=b", "/a/") match("/a/ =b", "/a/") match("/a/==b", "/a/") match("/a/ ==b", "/a/") match("/a/===b", "/a/") match("/a/ ===b", "/a/") match("/a/?b:c", "/a/") match("/a/ ? b : c", "/a/") match("/a/:c", "/a/") match("/a/g:c", "/a/g") match("/a/ : c", "/a/") match("/a/g : c", "/a/g") match("/a///", "/a/") match("/a/g//", "/a/g") match("/a/ //", "/a/") match("/a/g //", "/a/g") match("/a//**/", "/a/") match("/a/g/**/", "/a/g") match("/a/ /**/", "/a/") match("/a/g /**/", "/a/g") match("/a/g''", "/a/g") match("/a/g ''", "/a/g") match('/a/g""', "/a/g") match('/a/g ""', "/a/g") match("/a//b/", "/a/") match("/a/ /b/", "/a/") match("/a/g 0", "/a/g") match("/a/g 0.1", "/a/g") match("/a/g .1", "/a/g") match("/a/g 0x1", "/a/g") match("/a/g e", "/a/g") match("/a/g _", "/a/g") match("/a/g $", "/a/g") match("/a/g é", "/a/g") match("/a/g \\u0080", "/a/g") }) token("number", function(match) { match("1") match("1.") match("1..", "1.") match("0.1") match(".1") match("0.1.", "0.1") match("-1", false) match("-1.", false) match("-1..", false) match("-0.1", false) match("-.1", false) match("-0.1.", false) match("-", false) match("1e1") match("1.e1") match("1.e1.", "1.e1") match("0.1e1") match(".1e1") match("0.1e1.", "0.1e1") match("1e+1") match("1e-1") match("1e0123") match("1e0.123", "1e0") match("1e0x123", "1e0") match("1E1") match("1E+1") match("1E-1") match("1E0123") match("1E0.123", "1E0") match("1E0x123", "1E0") match("1E0o123", "1E0") match("1E0b123", "1E0") match("e1", false) match("e+1", false) match("e-1", false) match("E1", false) match("E+1", false) match("E-1", false) match("-e1", false) match("-e+1", false) match("-e-1", false) match("-E1", false) match("-E+1", false) match("-E-1", false) match("0x1") match("0xa") match("0x015cF") match("0x1e1") match("0x1E1") match("0x1g1", "0x1") match("0X1") match("0Xa") match("0X015cF") match("0X1e1") match("0X1E1") match("0X1g1", "0X1") match("-0x1", false) match("-0xa", false) match("-0x015cF", false) match("-0x1e1", false) match("-0x1E1", false) match("-0x1g1", false) match("0x", "0") match("1x1", "1") match("0x1.", "0x1") match("0x1.1", "0x1") match("0.0x1", "0.0") match(".0x1", ".0") match("0o1") match("0oa", "0") match("0o01574") match("0o1e1", "0o1") match("0o1E1", "0o1") match("0o1g1", "0o1") match("0O1") match("0Oa", "0") match("0O01574") match("0O1e1", "0O1") match("0O1E1", "0O1") match("0O1g1", "0O1") match("-0o1", false) match("-0oa", false) match("-0o01574", false) match("-0o1e1", false) match("-0o1E1", false) match("-0o1g1", false) match("0o", "0") match("1o1", "1") match("0o1.", "0o1") match("0o1.1", "0o1") match("0.0o1", "0.0") match(".0o1", ".0") match("0b1") match("0ba", "0") match("0b01011") match("0b1e1", "0b1") match("0b1E1", "0b1") match("0b1g1", "0b1") match("0B1") match("0Ba", "0") match("0B01011") match("0B1e1", "0B1") match("0B1E1", "0B1") match("0B1g1", "0B1") match("-0b1", false) match("-0ba", false) match("-0b01011", false) match("-0b1e1", false) match("-0b1E1", false) match("-0b1g1", false) match("0b", "0") match("1b1", "1") match("0b1.", "0b1") match("0b1.1", "0b1") match("0.0b1", "0.0") match(".0b1", ".0") }) token("name", function(match) { match("$") match("_") match("a") match("z") match("A") match("Z") match("å") match("π") match("0", false) match("0a", false) match("$0") match("_0") match("a0") match("z0") match("A0") match("Z0") match("å0") match("π0") match("a_56åπ") match("Iñtërnâtiônàlizætiøn☃💩") // The last character is Pile of poo. match("a\u00a0", "a") match("a\u1680", "a") match("a\u180e", "a") match("a\u2000", "a") match("a\u2001", "a") match("a\u2002", "a") match("a\u2003", "a") match("a\u2004", "a") match("a\u2005", "a") match("a\u2006", "a") match("a\u2007", "a") match("a\u2008", "a") match("a\u2009", "a") match("a\u200a", "a") match("a\u2028", "a") match("a\u2029", "a") match("a\u202f", "a") match("a\u205f", "a") match("a\u3000", "a") match("\\u0000") match("\\u15cF") match("\\u15cG", false) match("\\u000", false) match("\\u00000") match("a\\u0000b") match("\\u{0}") match("\\u{01}") match("\\u{012}") match("\\u{0123}") match("\\u{01234}") match("\\u{012345}") match("\\u{0123456}", false) match("\\u{15cF}") match("\\u{15cG}", false) match("a\\u{0000}b") match("\\x09", false) }) token("punctuator", function(match) { match("+") match("++") match("+=") match("++=", "++") match("-") match("--") match("-=") match("--=", "--") match("*") match("**") match("*=") match("**=") match("/") match("//", false) match("/=") match("//=", false) match("%") match("%%", "%") match("%=") match("%%=", "%") match("&") match("&&") match("&=") match("&&=", "&&") match("|") match("||") match("|=") match("||=", "||") match("^") match("^^", "^") match("^=") match("^^=", "^") match("<") match("<<") match("<<<", "<<") match("<=") match("<<=") match(">") match(">>") match(">>>") match(">=") match(">>=") match(">>>=") match("!") match("!=") match("!==") match("!===", "!==") match("=") match("==") match("===") match("=>") match("==>", "==") match("=>>", "=>") match("...") match("..", ".") match(".") match("....", "...") match("?") match("~") match(".") match(",") match(":") match(";") match("[") match("]") match("(") match(")") match("{") match("}") match("/a/()", "/") match("/a/g()", "/") match("/a/ ()", "/") match("/a/g ()", "/") match("/a/{}", "/") match("/a/g{}", "/") match("/a/ {}", "/") match("/a/g {}", "/") match("/a/+b", "/") match("/a/ +b", "/") match("/a/++b", "/") match("/a/ ++b", "/") match("/a/-b", "/") match("/a/ -b", "/") match("/a/--b", "/") match("/a/ --b", "/") match("/a/!b", "/") match("/a/ !b", "/") match("/a/~b", "/") match("/a/ ~b", "/") match("/a/g+b", "/") match("/a/g +b", "/") match("/a/g+=b", "/") match("/a/g +=b", "/") match("/a/g++", "/") match("/a/g ++", "/") match("/a/g-b", "/") match("/a/g -b", "/") match("/a/g-=b", "/") match("/a/g -=b", "/") match("/a/g--", "/") match("/a/g --", "/") match("/a/g*b", "/") match("/a/g *b", "/") match("/a/g *=b", "/") match("/a/g/b", "/") match("/a/g /b", "/") match("/a/g /=b", "/") match("/a/g%b", "/") match("/a/g %b", "/") match("/a/g%=b", "/") match("/a/g %=b", "/") match("/a/g&b", "/") match("/a/g &b", "/") match("/a/g&=b", "/") match("/a/g &=b", "/") match("/a/g&&b", "/") match("/a/g &&b", "/") match("/a/g|b", "/") match("/a/g |b", "/") match("/a/g|=b", "/") match("/a/g |=b", "/") match("/a/g||b", "/") match("/a/g ||b", "/") match("/a/g^b", "/") match("/a/g ^b", "/") match("/a/g^=b", "/") match("/a/g ^=b", "/") match("/a/gb", "/") match("/a/g >b", "/") match("/a/g>=b", "/") match("/a/g >=b", "/") match("/a/g>>b", "/") match("/a/g >>b", "/") match("/a/g>>=b", "/") match("/a/g >>=b", "/") match("/a/g>>>b", "/") match("/a/g >>>=b", "/") match("/a/g>>>=b", "/") match("/a/g >>>b", "/") match("/a/g!=b", "/") match("/a/g !=b", "/") match("/a/g!==b", "/") match("/a/g !==b", "/") match("/a/g=b", "/") match("/a/g =b", "/") match("/a/g==b", "/") match("/a/g ==b", "/") match("/a/g===b", "/") match("/a/g ===b", "/") match("/a/g?b:c", "/") match("/a/g ? b : c", "/") match("/a/''", "/") match("/a/ ''", "/") match('/a/""', "/") match('/a/ ""', "/") match("/a/g/b/", "/") match("/a/g /b/", "/") match("/a/0", "/") match("/a/ 0", "/") match("/a/0.1", "/") match("/a/ 0.1", "/") match("/a/.1", "/") match("/a/ .1", "/") match("/a/e", "/") match("/a/ e", "/") match("/a/_", "/") match("/a/ _", "/") match("/a/$", "/") match("/a/ $", "/") match("/a/é", "/") match("/a/ é", "/") match("/a/\\u0080", "/") match("/a/ \\u0080", "/") match("/a/ge", "/") match("/a/g_", "/") match("/a/g$", "/") match("/a/gé", "/") match("/a/g0", "/") match("/a/g\\u0080", "/") }) token("invalid", function(match) { match("") match("@") match("#") match("\\") match("\\xa9", "\\") match("\u0000") match("\u007F") }) }) suite("tokenization", function() { function testFile(file) { var contents = fs.readFileSync("test/fixtures/" + file + ".js").toString() var expected = require("./fixtures/" + file + ".json") var actual = contents.match(jsTokens) test(file + ".js", function() { assert.deepEqual(actual, expected) assert.equal(actual.join(""), contents) }) } testFile("base64") testFile("errors") testFile("regex") testFile("division") })