pax_global_header00006660000000000000000000000064121773342600014517gustar00rootroot0000000000000052 comment=f75c7db2e2a176f618ccd354e1897ed73d8fdb40 lex-parser-master/000077500000000000000000000000001217733426000144365ustar00rootroot00000000000000lex-parser-master/.gitignore000066400000000000000000000001001217733426000164150ustar00rootroot00000000000000lex-parser.js node_modules/ # Editor bak files *~ *.bak *.orig lex-parser-master/.npmignore000066400000000000000000000000141217733426000164300ustar00rootroot00000000000000lex.y lex.l lex-parser-master/Makefile000066400000000000000000000003701217733426000160760ustar00rootroot00000000000000 all: install build test install: npm install build: node ./node_modules/.bin/jison lex.y lex.l mv lex.js lex-parser.js test: node tests/all-tests.js clean: superclean: clean -find . -type d -name 'node_modules' -exec rm -rf "{}" \; lex-parser-master/README.md000066400000000000000000000102461217733426000157200ustar00rootroot00000000000000# lex-parser A parser for lexical grammars used by [jison](http://jison.org) and jison-lex. ## install npm install lex-parser ## build To build the parser yourself, clone the git repo then run: make This will generate `lex-parser.js`. ## usage var lexParser = require("lex-parser"); // parse a lexical grammar and return JSON lexParser.parse("%% ... "); ## example The parser can parse its own lexical grammar, shown below: NAME [a-zA-Z_][a-zA-Z0-9_-]* %s indented trail rules %x code start_condition options conditions action %% [^{}]+ return 'ACTION_BODY' "{" yy.depth++; return '{' "}" yy.depth == 0 ? this.begin('trail') : yy.depth--; return '}' {NAME} return 'NAME' ">" this.popState(); return '>' "," return ',' "*" return '*' \n+ /* */ \s+ this.begin('indented') "%%" this.begin('code'); return '%%' [a-zA-Z0-9_]+ return 'CHARACTER_LIT' {NAME} yy.options[yytext] = true \n+ this.begin('INITIAL') \s+\n+ this.begin('INITIAL') \s+ /* empty */ {NAME} return 'START_COND' \n+ this.begin('INITIAL') \s+\n+ this.begin('INITIAL') \s+ /* empty */ .*\n+ this.begin('rules') "{" yy.depth = 0; this.begin('action'); return '{' "%{"(.|\n)*?"%}" this.begin('trail'); yytext = yytext.substr(2, yytext.length-4);return 'ACTION' "%{"(.|\n)*?"%}" yytext = yytext.substr(2, yytext.length-4); return 'ACTION' .+ this.begin('rules'); return 'ACTION' "/*"(.|\n|\r)*?"*/" /* ignore */ "//".* /* ignore */ \n+ /* */ \s+ /* */ {NAME} return 'NAME' \"("\\\\"|'\"'|[^"])*\" yytext = yytext.replace(/\\"/g,'"');return 'STRING_LIT' "'"("\\\\"|"\'"|[^'])*"'" yytext = yytext.replace(/\\'/g,"'");return 'STRING_LIT' "|" return '|' "["("\\\\"|"\]"|[^\]])*"]" return 'ANY_GROUP_REGEX' "(?:" return 'SPECIAL_GROUP' "(?=" return 'SPECIAL_GROUP' "(?!" return 'SPECIAL_GROUP' "(" return '(' ")" return ')' "+" return '+' "*" return '*' "?" return '?' "^" return '^' "," return ',' "<>" return '$' "<" this.begin('conditions'); return '<' "/!" return '/!' "/" return '/' "\\"([0-7]{1,3}|[rfntvsSbBwWdD\\*+()${}|[\]\/.^?]|"c"[A-Z]|"x"[0-9A-F]{2}|"u"[a-fA-F0-9]{4}) return 'ESCAPE_CHAR' "\\". yytext = yytext.replace(/^\\/g,''); return 'ESCAPE_CHAR' "$" return '$' "." return '.' "%options" yy.options = {}; this.begin('options') "%s" this.begin('start_condition');return 'START_INC' "%x" this.begin('start_condition');return 'START_EXC' "%%" this.begin('rules'); return '%%' "{"\d+(","\s?\d+|",")?"}" return 'RANGE_REGEX' "{"{NAME}"}" return 'NAME_BRACE' "{" return '{' "}" return '}' . /* ignore bad characters */ <*><> return 'EOF' (.|\n)+ return 'CODE' %% ## license MIT lex-parser-master/lex.l000066400000000000000000000103321217733426000154020ustar00rootroot00000000000000 NAME [a-zA-Z_][a-zA-Z0-9_-]* BR \r\n|\n|\r %s indented trail rules %x code start_condition options conditions action %% "/*"(.|\n|\r)*?"*/" return 'ACTION_BODY'; "//".* return 'ACTION_BODY'; "/"[^ /]*?['"{}'][^ ]*?"/" return 'ACTION_BODY'; // regexp with braces or quotes (and no spaces) \"("\\\\"|'\"'|[^"])*\" return 'ACTION_BODY'; "'"("\\\\"|"\'"|[^'])*"'" return 'ACTION_BODY'; [/"'][^{}/"']+ return 'ACTION_BODY'; [^{}/"']+ return 'ACTION_BODY'; "{" yy.depth++; return '{' "}" yy.depth == 0 ? this.begin('trail') : yy.depth--; return '}' {NAME} return 'NAME'; ">" this.popState(); return '>'; "," return ','; "*" return '*'; {BR}+ /* */ \s+{BR}+ /* */ \s+ this.begin('indented') "%%" this.begin('code'); return '%%' [a-zA-Z0-9_]+ return 'CHARACTER_LIT' {NAME} yy.options[yytext] = true {BR}+ this.begin('INITIAL') \s+{BR}+ this.begin('INITIAL') \s+ /* empty */ {NAME} return 'START_COND' {BR}+ this.begin('INITIAL') \s+{BR}+ this.begin('INITIAL') \s+ /* empty */ .*{BR}+ this.begin('rules') "{" yy.depth = 0; this.begin('action'); return '{' "%{"(.|{BR})*?"%}" this.begin('trail'); yytext = yytext.substr(2, yytext.length-4);return 'ACTION' "%{"(.|{BR})*?"%}" yytext = yytext.substr(2, yytext.length-4); return 'ACTION' .+ this.begin('rules'); return 'ACTION' "/*"(.|\n|\r)*?"*/" /* ignore */ "//".* /* ignore */ {BR}+ /* */ \s+ /* */ {NAME} return 'NAME'; \"("\\\\"|'\"'|[^"])*\" yytext = yytext.replace(/\\"/g,'"'); return 'STRING_LIT'; "'"("\\\\"|"\'"|[^'])*"'" yytext = yytext.replace(/\\'/g,"'"); return 'STRING_LIT'; "|" return '|'; "["("\\\\"|"\]"|[^\]])*"]" return 'ANY_GROUP_REGEX'; "(?:" return 'SPECIAL_GROUP'; "(?=" return 'SPECIAL_GROUP'; "(?!" return 'SPECIAL_GROUP'; "(" return '('; ")" return ')'; "+" return '+'; "*" return '*'; "?" return '?'; "^" return '^'; "," return ','; "<>" return '$'; "<" this.begin('conditions'); return '<'; "/!" return '/!'; "/" return '/'; "\\"([0-7]{1,3}|[rfntvsSbBwWdD\\*+()${}|[\]\/.^?]|"c"[A-Z]|"x"[0-9A-F]{2}|"u"[a-fA-F0-9]{4}) return 'ESCAPE_CHAR'; "\\". yytext = yytext.replace(/^\\/g,''); return 'ESCAPE_CHAR'; "$" return '$'; "." return '.'; "%options" yy.options = {}; this.begin('options'); "%s" this.begin('start_condition'); return 'START_INC'; "%x" this.begin('start_condition'); return 'START_EXC'; "%%" this.begin('rules'); return '%%'; "{"\d+(","\s?\d+|",")?"}" return 'RANGE_REGEX'; "{"{NAME}"}" return 'NAME_BRACE'; "{" return '{'; "}" return '}'; . /* ignore bad characters */ <*><> return 'EOF'; (.|{BR})+ return 'CODE'; %% lex-parser-master/lex.y000066400000000000000000000104131217733426000154170ustar00rootroot00000000000000%start lex /* Jison lexer file format grammar */ %nonassoc '/' '/!' %left '*' '+' '?' RANGE_REGEX %% lex : definitions '%%' rules epilogue { $$ = { rules: $rules }; if ($definitions[0]) $$.macros = $definitions[0]; if ($definitions[1]) $$.startConditions = $definitions[1]; if ($epilogue) $$.moduleInclude = $epilogue; if (yy.options) $$.options = yy.options; if (yy.actionInclude) $$.actionInclude = yy.actionInclude; delete yy.options; delete yy.actionInclude; return $$; } ; epilogue : EOF { $$ = null; } | '%%' EOF { $$ = null; } | '%%' CODE EOF { $$ = $2; } ; definitions : definition definitions { $$ = $definitions; if ('length' in $definition) { $$[0] = $$[0] || {}; $$[0][$definition[0]] = $definition[1]; } else { $$[1] = $$[1] || {}; for (var name in $definition) { $$[1][name] = $definition[name]; } } } | ACTION definitions { yy.actionInclude += $1; $$ = $definitions; } | { yy.actionInclude = ''; $$ = [null,null]; } ; definition : NAME regex { $$ = [$1, $2]; } | START_INC names_inclusive { $$ = $2; } | START_EXC names_exclusive { $$ = $2; } ; names_inclusive : START_COND { $$ = {}; $$[$1] = 0; } | names_inclusive START_COND { $$ = $1; $$[$2] = 0; } ; names_exclusive : START_COND { $$ = {}; $$[$1] = 1; } | names_exclusive START_COND { $$ = $1; $$[$2] = 1; } ; rules : rules rule { $$ = $1; $$.push($2); } | rule { $$ = [$1]; } ; rule : start_conditions regex action { $$ = $1 ? [$1, $2, $3] : [$2,$3]; } ; action : '{' action_body '}' {$$ = $2;} | ACTION {$$ = $1;} ; action_body : {$$ = '';} | action_comments_body {$$ = $1;} | action_body '{' action_body '}' action_comments_body {$$ = $1+$2+$3+$4+$5;} | action_body '{' action_body '}' {$$ = $1 + $2 + $3 + $4;} ; action_comments_body : ACTION_BODY { $$ = yytext; } | action_comments_body ACTION_BODY { $$ = $1+$2; } ; start_conditions : '<' name_list '>' { $$ = $2; } | '<' '*' '>' { $$ = ['*']; } | ; name_list : NAME { $$ = [$1]; } | name_list ',' NAME { $$ = $1; $$.push($3); } ; regex : regex_list { $$ = $1; if (!(yy.options && yy.options.flex) && $$.match(/[\w\d]$/) && !$$.match(/\\(r|f|n|t|v|s|b|c[A-Z]|x[0-9A-F]{2}|u[a-fA-F0-9]{4}|[0-7]{1,3})$/)) { $$ += "\\b"; } } ; regex_list : regex_list '|' regex_concat { $$ = $1 + '|' + $3; } | regex_list '|' { $$ = $1 + '|'; } | regex_concat | { $$ = '' } ; regex_concat : regex_concat regex_base { $$ = $1 + $2; } | regex_base ; regex_base : '(' regex_list ')' { $$ = '(' + $2 + ')'; } | SPECIAL_GROUP regex_list ')' { $$ = $1 + $2 + ')'; } | regex_base '+' { $$ = $1 + '+'; } | regex_base '*' { $$ = $1 + '*'; } | regex_base '?' { $$ = $1 + '?'; } | '/' regex_base { $$ = '(?=' + $2 + ')'; } | '/!' regex_base { $$ = '(?!' + $2 + ')'; } | name_expansion | regex_base range_regex { $$ = $1 + $2; } | any_group_regex | '.' { $$ = '.'; } | '^' { $$ = '^'; } | '$' { $$ = '$'; } | string | escape_char ; name_expansion : NAME_BRACE ; any_group_regex : ANY_GROUP_REGEX { $$ = yytext; } ; escape_char : ESCAPE_CHAR { $$ = yytext; } ; range_regex : RANGE_REGEX { $$ = yytext; } ; string : STRING_LIT { $$ = prepareString(yytext.substr(1, yytext.length - 2)); } | CHARACTER_LIT ; %% function encodeRE (s) { return s.replace(/([.*+?^${}()|[\]\/\\])/g, '\\$1').replace(/\\\\u([a-fA-F0-9]{4})/g,'\\u$1'); } function prepareString (s) { // unescape slashes s = s.replace(/\\\\/g, "\\"); s = encodeRE(s); return s; }; lex-parser-master/package.json000066400000000000000000000007341217733426000167300ustar00rootroot00000000000000{ "name": "lex-parser", "version": "0.1.4", "description": "A parser for lexical grammars used by jison", "main": "lex-parser.js", "scripts": { "test": "make test" }, "repository": { "type": "git", "repository": "https://github.com/zaach/lex-parser.git" }, "keywords": [ "lexical", "grammar", "parser", "jison" ], "author": "Zach Carter", "license": "MIT", "devDependencies": { "jison": "0.4.x", "test": "*" } } lex-parser-master/tests/000077500000000000000000000000001217733426000156005ustar00rootroot00000000000000lex-parser-master/tests/all-tests.js000066400000000000000000000277731217733426000200660ustar00rootroot00000000000000var assert = require("assert"), lex = require("../lex-parser"), fs = require('fs'), path = require('path'); function read (p, file) { return fs.readFileSync(path.join(__dirname, p, file), "utf8"); } exports["test lex grammar with macros"] = function () { var lexgrammar = 'D [0-9]\nID [a-zA-Z][a-zA-Z0-9]+\n%%\n\n{D}"ohhai" {print(9);}\n"{" return \'{\';'; var expected = { macros: {"D": "[0-9]", "ID": "[a-zA-Z][a-zA-Z0-9]+"}, rules: [ ["{D}ohhai\\b", "print(9);"], ["\\{", "return '{';"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test escaped chars"] = function () { var lexgrammar = '%%\n"\\n"+ {return \'NL\';}\n\\n+ {return \'NL2\';}\n\\s+ {/* skip */}'; var expected = { rules: [ ["\\\\n+", "return 'NL';"], ["\\n+", "return 'NL2';"], ["\\s+", "/* skip */"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test advanced"] = function () { var lexgrammar = '%%\n$ {return \'EOF\';}\n. {/* skip */}\n"stuff"*/("{"|";") {/* ok */}\n(.+)[a-z]{1,2}"hi"*? {/* skip */}\n'; var expected = { rules: [ ["$", "return 'EOF';"], [".", "/* skip */"], ["stuff*(?=(\\{|;))", "/* ok */"], ["(.+)[a-z]{1,2}hi*?", "/* skip */"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test [^\\]]"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {return true;}\n\'f"oo\\\'bar\' {return \'baz2\';}\n"fo\\"obar" {return \'baz\';}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "return true;"], ["f\"oo'bar\\b", "return 'baz2';"], ['fo"obar\\b', "return 'baz';"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" %{\nreturn true;\n%}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nreturn true;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action with single braces"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {\nvar b={};return true;\n}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nvar b={};return true;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action with brace in a multi-line-comment"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {\nvar b={}; /* { */ return true;\n}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nvar b={}; /* { */ return true;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action with brace in a single-line-comment"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {\nvar b={}; // { \nreturn 2 / 3;\n}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nvar b={}; // { \nreturn 2 / 3;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action with braces in strings"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {\nvar b=\'{\' + "{"; // { \nreturn 2 / 3;\n}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nvar b='{' + \"{\"; // { \nreturn 2 / 3;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test multiline action with braces in regexp"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" {\nvar b=/{/; // { \nreturn 2 / 3;\n}\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\nvar b=/{/; // { \nreturn 2 / 3;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test include"] = function () { var lexgrammar = '\nRULE [0-9]\n\n%{\n hi \n%}\n%%\n"["[^\\]]"]" %{\nreturn true;\n%}\n'; var expected = { macros: {"RULE": "[0-9]"}, actionInclude: "\n hi \n", rules: [ ["\\[[^\\]]\\]", "\nreturn true;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test bnf lex grammar"] = function () { var lexgrammar = lex.parse(read('lex', 'bnf.jisonlex')); var expected = JSON.parse(read('lex', 'bnf.lex.json')); assert.deepEqual(lexgrammar, expected, "grammar should be parsed correctly"); }; exports["test lex grammar bootstrap"] = function () { var lexgrammar = lex.parse(read('lex', 'lex_grammar.jisonlex')); var expected = JSON.parse(read('lex', 'lex_grammar.lex.json')); assert.deepEqual(lexgrammar, expected, "grammar should be parsed correctly"); }; exports["test ANSI C lexical grammar"] = function () { var lexgrammar = lex.parse(read('lex','ansic.jisonlex')); assert.ok(lexgrammar, "grammar should be parsed correctly"); }; exports["test advanced"] = function () { var lexgrammar = '%%\n"stuff"*/!("{"|";") {/* ok */}\n'; var expected = { rules: [ ["stuff*(?!(\\{|;))", "/* ok */"], ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test start conditions"] = function () { var lexgrammar = '%s TEST TEST2\n%x EAT\n%%\n'+ '"enter-test" {this.begin(\'TEST\');}\n'+ '"x" {return \'T\';}\n'+ '<*>"z" {return \'Z\';}\n'+ '"y" {this.begin(\'INITIAL\'); return \'TY\';}'; var expected = { startConditions: { "TEST": 0, "TEST2": 0, "EAT": 1, }, rules: [ ["enter-test\\b", "this.begin('TEST');" ], [["TEST","EAT"], "x\\b", "return 'T';" ], [["*"], "z\\b", "return 'Z';" ], [["TEST"], "y\\b", "this.begin('INITIAL'); return 'TY';" ] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test no brace action"] = function () { var lexgrammar = '%%\n"["[^\\]]"]" return true;\n"x" return 1;'; var expected = { rules: [ ["\\[[^\\]]\\]", "return true;"], ["x\\b", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test quote escape"] = function () { var lexgrammar = '%%\n\\"\\\'"x" return 1;'; var expected = { rules: [ ["\"'x\\b", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test escape things"] = function () { var lexgrammar = '%%\n\\"\\\'\\\\\\*\\i return 1;\n"a"\\b return 2;\n\\cA {}\n\\012 {}\n\\xFF {}'; var expected = { rules: [ ["\"'\\\\\\*i\\b", "return 1;"], ["a\\b", "return 2;"], ["\\cA", ""], ["\\012", ""], ["\\xFF", ""] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test unicode encoding"] = function () { var lexgrammar = '%%\n"\\u03c0" return 1;'; var expected = { rules: [ ["\\u03c0", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test unicode"] = function () { var lexgrammar = '%%\n"π" return 1;'; var expected = { rules: [ ["π", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test bugs"] = function () { var lexgrammar = '%%\n\\\'([^\\\\\']+|\\\\(\\n|.))*?\\\' return 1;'; var expected = { rules: [ ["'([^\\\\']+|\\\\(\\n|.))*?'", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test special groupings"] = function () { var lexgrammar = '%%\n(?:"foo"|"bar")\\(\\) return 1;'; var expected = { rules: [ ["(?:foo|bar)\\(\\)", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test trailing code include"] = function () { var lexgrammar = '%%"foo" {return bar;}\n%% var bar = 1;'; var expected = { rules: [ ['foo\\b', "return bar;"] ], moduleInclude: " var bar = 1;" }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test empty or regex"] = function () { var lexgrammar = '%%\n(|"bar")("foo"|)(|) return 1;'; var expected = { rules: [ ["(|bar)(foo|)(|)", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test options"] = function () { var lexgrammar = '%options flex\n%%\n"foo" return 1;'; var expected = { rules: [ ["foo", "return 1;"] ], options: {flex: true} }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test unquoted string rules"] = function () { var lexgrammar = "%%\nfoo* return 1"; var expected = { rules: [ ["foo*", "return 1"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test [^\\\\]"] = function () { var lexgrammar = '%%\n"["[^\\\\]"]" {return true;}\n\'f"oo\\\'bar\' {return \'baz2\';}\n"fo\\"obar" {return \'baz\';}\n'; var expected = { rules: [ ["\\[[^\\\\]\\]", "return true;"], ["f\"oo'bar\\b", "return 'baz2';"], ['fo"obar\\b', "return 'baz';"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test comments"] = function () { var lexgrammar = "/* */ // foo\n%%\nfoo* return 1"; var expected = { rules: [ ["foo*", "return 1"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test rules with trailing escapes"] = function () { var lexgrammar = '%%\n\\#[^\\n]*\\n {/* ok */}\n'; var expected = { rules: [ ["#[^\\n]*\\n", "/* ok */"], ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test no brace action with surplus whitespace between rules"] = function () { var lexgrammar = '%%\n"a" return true;\n \n"b" return 1;\n \n'; var expected = { rules: [ ["a\\b", "return true;"], ["b\\b", "return 1;"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test windows line endings"] = function () { var lexgrammar = '%%\r\n"["[^\\]]"]" %{\r\nreturn true;\r\n%}\r\n'; var expected = { rules: [ ["\\[[^\\]]\\]", "\r\nreturn true;\r\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; exports["test braced action with surplus whitespace between rules"] = function () { var lexgrammar = '%%\n"a" %{ \nreturn true;\n%} \n \n"b" %{ return 1;\n%} \n \n'; var expected = { rules: [ ["a\\b", " \nreturn true;\n"], ["b\\b", " return 1;\n"] ] }; assert.deepEqual(lex.parse(lexgrammar), expected, "grammar should be parsed correctly"); }; if (require.main === module) require("test").run(exports); lex-parser-master/tests/lex/000077500000000000000000000000001217733426000163705ustar00rootroot00000000000000lex-parser-master/tests/lex/ansic.jisonlex000066400000000000000000000067251217733426000212540ustar00rootroot00000000000000D [0-9] L [a-zA-Z_] H [a-fA-F0-9] E [Ee][+-]?{D}+ FS [fFlL] IS [uUlL]* %{ #include #include "y.tab.h" void count(); %} %% "/*" { comment(); } "auto" { count(); return(AUTO); } "break" { count(); return(BREAK); } "case" { count(); return(CASE); } "char" { count(); return(CHAR); } "const" { count(); return(CONST); } "continue" { count(); return(CONTINUE); } "default" { count(); return(DEFAULT); } "do" { count(); return(DO); } "double" { count(); return(DOUBLE); } "else" { count(); return(ELSE); } "enum" { count(); return(ENUM); } "extern" { count(); return(EXTERN); } "float" { count(); return(FLOAT); } "for" { count(); return(FOR); } "goto" { count(); return(GOTO); } "if" { count(); return(IF); } "int" { count(); return(INT); } "long" { count(); return(LONG); } "register" { count(); return(REGISTER); } "return" { count(); return(RETURN); } "short" { count(); return(SHORT); } "signed" { count(); return(SIGNED); } "sizeof" { count(); return(SIZEOF); } "static" { count(); return(STATIC); } "struct" { count(); return(STRUCT); } "switch" { count(); return(SWITCH); } "typedef" { count(); return(TYPEDEF); } "union" { count(); return(UNION); } "unsigned" { count(); return(UNSIGNED); } "void" { count(); return(VOID); } "volatile" { count(); return(VOLATILE); } "while" { count(); return(WHILE); } {L}({L}|{D})* { count(); return(check_type()); } "0"[xX]{H}+{IS}? { count(); return(CONSTANT); } "0"{D}+{IS}? { count(); return(CONSTANT); } {D}+{IS}? { count(); return(CONSTANT); } "L"?"'"("\'"|[^'])+"'" { count(); return(CONSTANT); } {D}+{E}{FS}? { count(); return(CONSTANT); } {D}*"."{D}+({E})?{FS}? { count(); return(CONSTANT); } {D}+"."{D}*({E})?{FS}? { count(); return(CONSTANT); } "L"?'"'('\"'|[^"])*'"' { count(); return(STRING_LITERAL); } "..." { count(); return(ELLIPSIS); } ">>=" { count(); return(RIGHT_ASSIGN); } "<<=" { count(); return(LEFT_ASSIGN); } "+=" { count(); return(ADD_ASSIGN); } "-=" { count(); return(SUB_ASSIGN); } "*=" { count(); return(MUL_ASSIGN); } "/=" { count(); return(DIV_ASSIGN); } "%=" { count(); return(MOD_ASSIGN); } "&=" { count(); return(AND_ASSIGN); } "^=" { count(); return(XOR_ASSIGN); } "|=" { count(); return(OR_ASSIGN); } ">>" { count(); return(RIGHT_OP); } "<<" { count(); return(LEFT_OP); } "++" { count(); return(INC_OP); } "--" { count(); return(DEC_OP); } "->" { count(); return(PTR_OP); } "&&" { count(); return(AND_OP); } "||" { count(); return(OR_OP); } "<=" { count(); return(LE_OP); } ">=" { count(); return(GE_OP); } "==" { count(); return(EQ_OP); } "!=" { count(); return(NE_OP); } ";" { count(); return(';'); } ("{"|"<%") { count(); return('{'); } ("}"|"%>") %{ count(); return('}'); %} "," { count(); return(','); } ":" { count(); return(':'); } "=" { count(); return('='); } "(" { count(); return('('); } ")" { count(); return(')'); } ("["|"<:") { count(); return('['); } ("]"|":>") { count(); return(']'); } "." { count(); return('.'); } "&" { count(); return('&'); } "!" { count(); return('!'); } "~" { count(); return('~'); } "-" { count(); return('-'); } "+" { count(); return('+'); } "*" { count(); return('*'); } "/" { count(); return('/'); } "%" { count(); return('%'); } "<" { count(); return('<'); } ">" { count(); return('>'); } "^" { count(); return('^'); } "|" { count(); return('|'); } "?" { count(); return('?'); } [ \t\v\n\f] { count(); } . { /* ignore bad characters */ } %% lex-parser-master/tests/lex/bnf.jisonlex000066400000000000000000000013771217733426000207220ustar00rootroot00000000000000 %% \s+ {/* skip whitespace */} "/*"[^*]*"*" {return yy.lexComment(this);} [a-zA-Z][a-zA-Z0-9_-]* {return 'ID';} '"'[^"]+'"' {yytext = yytext.substr(1, yyleng-2); return 'STRING';} "'"[^']+"'" {yytext = yytext.substr(1, yyleng-2); return 'STRING';} ":" {return ':';} ";" {return ';';} "|" {return '|';} "%%" {return '%%';} "%prec" {return 'PREC';} "%start" {return 'START';} "%left" {return 'LEFT';} "%right" {return 'RIGHT';} "%nonassoc" {return 'NONASSOC';} "%"[a-zA-Z]+[^\n]* {/* ignore unrecognized decl */} "{{"[^}]*"}" {return yy.lexAction(this);} "{"[^}]*"}" {yytext = yytext.substr(1, yyleng-2); return 'ACTION';} "<"[^>]*">" {yytext = yytext.substr(1, yyleng-2); return 'ACTION';} . {/* ignore bad characters */} <> {return 'EOF';} lex-parser-master/tests/lex/bnf.lex.json000066400000000000000000000017421217733426000206230ustar00rootroot00000000000000{ "rules": [ ["\\s+", "/* skip whitespace */"], ["\\/\\*[^*]*\\*", "return yy.lexComment(this);"], ["[a-zA-Z][a-zA-Z0-9_-]*", "return 'ID';"], ["\"[^\"]+\"", "yytext = yytext.substr(1, yyleng-2); return 'STRING';"], ["'[^']+'", "yytext = yytext.substr(1, yyleng-2); return 'STRING';"], [":", "return ':';"], [";", "return ';';"], ["\\|", "return '|';"], ["%%", "return '%%';"], ["%prec\\b", "return 'PREC';"], ["%start\\b", "return 'START';"], ["%left\\b", "return 'LEFT';"], ["%right\\b", "return 'RIGHT';"], ["%nonassoc\\b", "return 'NONASSOC';"], ["%[a-zA-Z]+[^\\n]*", "/* ignore unrecognized decl */"], ["\\{\\{[^}]*\\}", "return yy.lexAction(this);"], ["\\{[^}]*\\}", "yytext = yytext.substr(1, yyleng-2); return 'ACTION';"], ["<[^>]*>", "yytext = yytext.substr(1, yyleng-2); return 'ACTION';"], [".", "/* ignore bad characters */"], ["$", "return 'EOF';"] ] } lex-parser-master/tests/lex/lex_grammar.jisonlex000066400000000000000000000016741217733426000224530ustar00rootroot00000000000000 %% \n+ {yy.freshLine = true;} \s+ {yy.freshLine = false;} "y{"[^}]*"}" {yytext = yytext.substr(2, yytext.length-3);return 'ACTION';} [a-zA-Z_][a-zA-Z0-9_-]* {return 'NAME';} '"'([^"]|'\"')*'"' {return 'STRING_LIT';} "'"([^']|"\'")*"'" {return 'STRING_LIT';} "|" {return '|';} "["("\]"|[^\]])*"]" {return 'ANY_GROUP_REGEX';} "(" {return '(';} ")" {return ')';} "+" {return '+';} "*" {return '*';} "?" {return '?';} "^" {return '^';} "/" {return '/';} "\\"[a-zA-Z0] {return 'ESCAPE_CHAR';} "$" {return '$';} "<>" {return '$';} "." {return '.';} "%%" {return '%%';} "{"\d+(","\s?\d+|",")?"}" {return 'RANGE_REGEX';} /"{" %{if(yy.freshLine){this.input('{');return '{';} else this.unput('y');%} "}" %{return '}';%} "%{"(.|\n)*?"}%" {yytext = yytext.substr(2, yytext.length-4);return 'ACTION';} . {/* ignore bad characters */} <> {return 'EOF';} lex-parser-master/tests/lex/lex_grammar.lex.json000066400000000000000000000023141217733426000223500ustar00rootroot00000000000000{ "rules": [ ["\\n+", "yy.freshLine = true;"], ["\\s+", "yy.freshLine = false;"], ["y\\{[^}]*\\}", "yytext = yytext.substr(2, yytext.length-3);return 'ACTION';"], ["[a-zA-Z_][a-zA-Z0-9_-]*", "return 'NAME';"], ["\"([^\"]|\\\\\")*\"", "return 'STRING_LIT';"], ["'([^']|\\\\')*'", "return 'STRING_LIT';"], ["\\|", "return '|';"], ["\\[(\\\\\\]|[^\\]])*\\]", "return 'ANY_GROUP_REGEX';"], ["\\(", "return '(';"], ["\\)", "return ')';"], ["\\+", "return '+';"], ["\\*", "return '*';"], ["\\?", "return '?';"], ["\\^", "return '^';"], ["\\/", "return '/';"], ["\\\\[a-zA-Z0]", "return 'ESCAPE_CHAR';"], ["\\$", "return '$';"], ["<>", "return '$';"], ["\\.", "return '.';"], ["%%", "return '%%';"], ["\\{\\d+(,\\s?\\d+|,)?\\}", "return 'RANGE_REGEX';"], ["(?=\\{)", "if(yy.freshLine){this.input('{');return '{';} else this.unput('y');"], ["\\}", "return '}';"], ["%\\{(.|\\n)*?\\}%", "yytext = yytext.substr(2, yytext.length-4);return 'ACTION';"], [".", "/* ignore bad characters */"], ["$", "return 'EOF';"] ] }