pax_global_header 0000666 0000000 0000000 00000000064 13113476405 0014516 g ustar 00root root 0000000 0000000 52 comment=ba60153ecc8ae6ef56d306a6cdb6c906602e6aef
neotoma-1.7.4+dfsg/ 0000775 0000000 0000000 00000000000 13113476405 0014110 5 ustar 00root root 0000000 0000000 neotoma-1.7.4+dfsg/.gitignore 0000664 0000000 0000000 00000000156 13113476405 0016102 0 ustar 00root root 0000000 0000000 ebin/*.beam
ebin/neotoma.app
extra/*.beam
*#*
.eunit
*#*
.DS_Store
._*
neotoma.plt
/_build/
/doc/
/rebar.lock
neotoma-1.7.4+dfsg/.travis.yml 0000664 0000000 0000000 00000000227 13113476405 0016222 0 ustar 00root root 0000000 0000000 language: erlang
notifications:
disabled: true
otp_release:
- 17.0
- 17.3
- 17.4
- R16B02
- R16B01
- R16B
- R15B02
- R15B01
- R15B
neotoma-1.7.4+dfsg/LICENSE 0000664 0000000 0000000 00000002036 13113476405 0015116 0 ustar 00root root 0000000 0000000 Copyright (c) 2009 Sean Cribbs
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in
all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
THE SOFTWARE. neotoma-1.7.4+dfsg/Makefile 0000664 0000000 0000000 00000001010 13113476405 0015540 0 ustar 00root root 0000000 0000000 .PHONY: test compile clean dialyzer bootstrap escript
all: compile
compile:
@./rebar compile
test:
@./rebar eunit
clean:
@./rebar clean
neotoma.plt:
@dialyzer --build_plt --apps erts kernel stdlib compiler crypto hipe \
syntax_tools --output_plt neotoma.plt
dialyzer: compile neotoma.plt
@dialyzer --plt neotoma.plt ebin
xref: compile
@./rebar xref skip_deps=true
bootstrap: compile
@erl -pz ebin -b start_sasl -noshell -s init stop -s neotoma bootstrap
@./rebar compile
escript:
@./rebar escriptize
neotoma-1.7.4+dfsg/README.textile 0000664 0000000 0000000 00000003763 13113476405 0016456 0 ustar 00root root 0000000 0000000 h1. Neotoma
!https://travis-ci.org/seancribbs/neotoma.svg?branch=master!:https://travis-ci.org/seancribbs/neotoma
h2. About
Neotoma is a packrat parser-generator for Erlang for Parsing Expression Grammars (PEGs).
It consists of a parsing-combinator library with memoization routines, a parser for PEGs,
and a utility to generate parsers from PEGs. It is inspired by treetop, a Ruby library with
similar aims, and parsec, the parser-combinator library for Haskell.
Neotoma is licensed under the MIT License (see LICENSE).
h2. Features
# Simple, declarative parsers generated from even simpler grammars.
# Fully integrated, single-pass lexical and syntactic analysis (a feature of PEGs).
# Packrat-style memoization, boasting parse-time bound linearly to the input size (at the expense of memory usage).
# In-place semantic analysis/transformation, supporting single-pass end-to-end in some applications.
# Erlang code-generation for the lexical/syntactic analysis piece, with the option of semantic analysis/transformation inline, or in a separate module.
# Line/column number tracking for easy resolution of parsing errors.
h2. Installation
# Clone the repository from github: $ git clone git://github.com/seancribbs/neotoma.git
# Symlink or copy the cloned repository to somewhere in your Erlang code path. $ERLANG_HOME/lib is best.
# Build the source: $ make
h2. Usage
# After making sure the library is in your code path, fire up an Erlang shell.
# To generate a parser from a PEG, use @neotoma:file/1,2@ (more detailed documentation pending). For PEG examples, see the @extra/@ directory in the repository.
1> neotoma:file("extra/arithmetic.peg").
This will place @arithmetic.erl@ in the same directory as the @.peg@ file by default.
h2. Contributing
Please send pull-requests to 'seancribbs' on github. When submitting a patch, eunit tests are strongly encouraged.
neotoma-1.7.4+dfsg/extra/ 0000775 0000000 0000000 00000000000 13113476405 0015233 5 ustar 00root root 0000000 0000000 neotoma-1.7.4+dfsg/extra/arithmetic.erl 0000664 0000000 0000000 00000014252 13113476405 0020074 0 ustar 00root root 0000000 0000000 -module(arithmetic).
-export([parse/1,file/1]).
-compile(nowarn_unused_vars).
-compile({nowarn_unused_function,[p/4, p/5, p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1, line/1, column/1]}).
file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(binary_to_list(Bin)).
parse(Input) ->
setup_memo(),
Result = case 'additive'(Input,{{line,1},{column,1}}) of
{AST, [], _Index} -> AST;
Any -> Any
end,
release_memo(), Result.
'additive'(Input, Index) ->
p(Input, Index, 'additive', fun(I,D) -> (p_choose([p_seq([fun 'multitive'/2, p_string("+"), fun 'additive'/2]), fun 'multitive'/2]))(I,D) end, fun(Node, Idx) ->
case Node of
Int when is_integer(Int) -> Int;
[A, "+", B] -> A + B
end
end).
'multitive'(Input, Index) ->
p(Input, Index, 'multitive', fun(I,D) -> (p_choose([p_seq([fun 'primary'/2, p_string("*"), fun 'multitive'/2]), fun 'primary'/2]))(I,D) end, fun(Node, Idx) ->
case Node of
Int when is_integer(Int) -> Int;
[A,"*",B] -> A * B
end
end).
'primary'(Input, Index) ->
p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([p_string("("), fun 'additive'/2, p_string(")")]), fun 'decimal'/2]))(I,D) end, fun(Node, Idx) ->
case Node of
Int when is_integer(Int) -> Int;
List when is_list(List) -> lists:nth(2,List)
end
end).
'decimal'(Input, Index) ->
p(Input, Index, 'decimal', fun(I,D) -> (p_one_or_more(p_charclass("[0-9]")))(I,D) end, fun(Node, Idx) -> list_to_integer(Node) end).
p(Inp, Index, Name, ParseFun) ->
p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end).
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
% Grab the memo table from ets
Memo = get_memo(StartIndex),
% See if the current reduction is memoized
case dict:find(Name, Memo) of
% If it is, return the result
{ok, Result} -> Result;
% If not, attempt to parse
_ ->
case ParseFun(Inp, StartIndex) of
% If it fails, memoize the failure
{fail,_} = Failure ->
memoize(StartIndex, dict:store(Name, Failure, Memo)),
Failure;
% If it passes, transform and memoize the result.
{Result, InpRem, NewIndex} ->
Transformed = TransformFun(Result, StartIndex),
memoize(StartIndex, dict:store(Name, {Transformed, InpRem, NewIndex}, Memo)),
{Transformed, InpRem, NewIndex}
end
end.
setup_memo() ->
put(parse_memo_table, ets:new(?MODULE, [set])).
release_memo() ->
ets:delete(memo_table_name()).
memoize(Position, Struct) ->
ets:insert(memo_table_name(), {Position, Struct}).
get_memo(Position) ->
case ets:lookup(memo_table_name(), Position) of
[] -> dict:new();
[{Position, Dict}] -> Dict
end.
memo_table_name() ->
get(parse_memo_table).
p_eof() ->
fun([], Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
p_and(P) ->
p_seq(P).
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
p_string(S) ->
fun(Input, Index) ->
case lists:prefix(S, Input) of
true -> {S, lists:sublist(Input, length(S)+1, length(Input)), p_advance_index(S,Index)};
_ -> {fail, {expected, {string, S}, Index}}
end
end.
p_anything() ->
fun([], Index) -> {fail, {expected, any_character, Index}};
([H|T], Index) -> {H, T, p_advance_index(H, Index)}
end.
p_charclass(Class) ->
fun(Inp, Index) ->
{ok, RE} = re:compile("^"++Class),
case re:run(Inp, RE) of
{match, _} ->
{hd(Inp), tl(Inp), p_advance_index(hd(Inp), Index)};
_ -> {fail,{expected, {character_class, Class}, Index}}
end
end.
line({{line,L},_}) -> L;
line(_) -> undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) -> % strings
lists:foldl(fun p_advance_index/2, Index, MatchedInput);
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/extra/arithmetic.peg 0000664 0000000 0000000 00000000721 13113476405 0020061 0 ustar 00root root 0000000 0000000 additive <- multitive "+" additive / multitive `
case Node of
Int when is_integer(Int) -> Int;
[A, "+", B] -> A + B
end
`;
multitive <- primary "*" Mul:multitive / primary `
case Node of
Int when is_integer(Int) -> Int;
[A,"*",{'Mul',B}] -> A * B
end
`;
primary <- par:("(" add:additive ")") / dec:Decimal `
case Node of
{dec,Int} when is_integer(Int) -> Int;
{par,List} -> proplists:get_value(add,List)
end
`;
Decimal <- [0-9]+ `list_to_integer(Node)`; neotoma-1.7.4+dfsg/extra/csv.erl 0000664 0000000 0000000 00000030137 13113476405 0016536 0 ustar 00root root 0000000 0000000 -module(csv).
-export([parse/1,file/1]).
-compile(nowarn_unused_vars).
-compile({nowarn_unused_function,[p/4, p/5, p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1, line/1, column/1]}).
file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(Bin).
parse(List) when is_list(List) -> parse(list_to_binary(List));
parse(Input) when is_binary(Input) ->
setup_memo(),
Result = case 'rows'(Input,{{line,1},{column,1}}) of
{AST, <<>>, _Index} -> AST;
Any -> Any
end,
release_memo(), Result.
'rows'(Input, Index) ->
p(Input, Index, 'rows', fun(I,D) -> (p_choose([p_seq([p_label('head', fun 'row'/2), p_label('tail', p_zero_or_more(p_seq([fun 'crlf'/2, fun 'row'/2])))]), p_string(<<>>)]))(I,D) end, fun(Node, Idx) ->
case Node of
[] -> [];
[""] -> [];
_ ->
Head = proplists:get_value(head, Node),
Tail = [R || [_,R] <- proplists:get_value(tail, Node)],
[Head|Tail]
end
end).
'row'(Input, Index) ->
p(Input, Index, 'row', fun(I,D) -> (p_choose([p_seq([p_label('head', fun 'field'/2), p_label('tail', p_zero_or_more(p_seq([fun 'field_sep'/2, fun 'field'/2])))]), p_string(<<>>)]))(I,D) end, fun(Node, Idx) ->
case Node of
[] -> [];
[""] -> [];
_ ->
Head = proplists:get_value(head, Node),
Tail = [F || [_,F] <- proplists:get_value(tail, Node)],
[Head|Tail]
end
end).
'field'(Input, Index) ->
p(Input, Index, 'field', fun(I,D) -> (p_choose([fun 'quoted_field'/2, p_zero_or_more(p_seq([p_not(fun 'field_sep'/2), p_not(fun 'crlf'/2), p_anything()]))]))(I,D) end, fun(Node, Idx) -> unicode:characters_to_binary(Node) end).
'quoted_field'(Input, Index) ->
p(Input, Index, 'quoted_field', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('string', p_zero_or_more(p_choose([p_string(<<"\"\"">>), p_seq([p_not(p_string(<<"\"">>)), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, Idx) ->
String = proplists:get_value(string, Node),
re:replace(String, "[\"]{2}", "\"",[global, {return, binary}])
end).
'field_sep'(Input, Index) ->
p(Input, Index, 'field_sep', fun(I,D) -> (p_string(<<",">>))(I,D) end, fun(Node, Idx) -> Node end).
'crlf'(Input, Index) ->
p(Input, Index, 'crlf', fun(I,D) -> (p_seq([p_optional(p_charclass(<<"[\r]">>)), p_charclass(<<"[\n]">>)]))(I,D) end, fun(Node, Idx) -> Node end).
p(Inp, Index, Name, ParseFun) ->
p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end).
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
% Grab the memo table from ets
Memo = get_memo(StartIndex),
% See if the current reduction is memoized
case proplists:lookup(Name, Memo) of
% If it is, return the result
{Name, Result} -> Result;
% If not, attempt to parse
_ ->
case ParseFun(Inp, StartIndex) of
% If it fails, memoize the failure
{fail,_} = Failure ->
memoize(StartIndex, [{Name, Failure}|Memo]),
Failure;
% If it passes, transform and memoize the result.
{Result, InpRem, NewIndex} ->
Transformed = TransformFun(Result, StartIndex),
memoize(StartIndex, [{Name, {Transformed, InpRem, NewIndex}}|Memo]),
{Transformed, InpRem, NewIndex}
end
end.
setup_memo() ->
put(parse_memo_table, ets:new(?MODULE, [set])).
release_memo() ->
ets:delete(memo_table_name()).
memoize(Position, Struct) ->
ets:insert(memo_table_name(), {Position, Struct}).
get_memo(Position) ->
case ets:lookup(memo_table_name(), Position) of
[] -> [];
[{Position, PList}] -> PList
end.
memo_table_name() ->
get(parse_memo_table).
p_eof() ->
fun(<<>>, Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
p_and(P) ->
p_seq(P).
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
p_string(S) when is_list(S) -> p_string(list_to_binary(S));
p_string(S) ->
Length = erlang:byte_size(S),
fun(Input, Index) ->
try
<> = Input,
{S, Rest, p_advance_index(S, Index)}
catch
error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
end
end.
p_anything() ->
fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
(Input, Index) when is_binary(Input) ->
<> = Input,
{<>, Rest, p_advance_index(<>, Index)}
end.
p_charclass(Class) ->
{ok, RE} = re:compile(Class, [unicode, dotall]),
fun(Inp, Index) ->
case re:run(Inp, RE, [anchored]) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {character_class, Class}, Index}}
end
end.
line({{line,L},_}) -> L;
line(_) -> undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/extra/csv.peg 0000664 0000000 0000000 00000001315 13113476405 0016523 0 ustar 00root root 0000000 0000000 rows <- head:row tail:(crlf row)* / ''
`
case Node of
[] -> [];
[""] -> [];
_ ->
Head = proplists:get_value(head, Node),
Tail = [R || [_,R] <- proplists:get_value(tail, Node)],
[Head|Tail]
end
`;
row <- head:field tail:(field_sep field)* / ''
`
case Node of
[] -> [];
[""] -> [];
_ ->
Head = proplists:get_value(head, Node),
Tail = [F || [_,F] <- proplists:get_value(tail, Node)],
[Head|Tail]
end
`;
field <- quoted_field / (!field_sep !crlf .)* `iolist_to_binary(Node)`;
quoted_field <- '"' string:('""' / (!'"' .))* '"'
`
String = proplists:get_value(string, Node),
re:replace(String, "[\"]{2}", "\"",[global, {return, binary}])
`;
field_sep <- ',' ~;
crlf <- [\r]? [\n] ~; neotoma-1.7.4+dfsg/extra/json.erl 0000664 0000000 0000000 00000043625 13113476405 0016722 0 ustar 00root root 0000000 0000000 -module(json).
-export([parse/1,file/1]).
-compile(nowarn_unused_vars).
-compile({nowarn_unused_function,[p/4, p/5, p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_and/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1, line/1, column/1]}).
file(Filename) -> {ok, Bin} = file:read_file(Filename), parse(Bin).
parse(List) when is_list(List) -> parse(list_to_binary(List));
parse(Input) when is_binary(Input) ->
setup_memo(),
Result = case 'json_value'(Input,{{line,1},{column,1}}) of
{AST, <<>>, _Index} -> AST;
Any -> Any
end,
release_memo(), Result.
'json_value'(Input, Index) ->
p(Input, Index, 'json_value', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), p_choose([fun 'object'/2, fun 'array'/2, fun 'string'/2, fun 'number'/2, fun 'true'/2, fun 'false'/2, fun 'null'/2]), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, Idx) -> lists:nth(2, Node) end).
'object'(Input, Index) ->
p(Input, Index, 'object', fun(I,D) -> (p_choose([p_seq([p_string(<<"{">>), p_optional(fun 'space'/2), p_label('head', fun 'pair'/2), p_label('tail', p_zero_or_more(p_seq([p_optional(fun 'space'/2), p_string(<<",">>), p_optional(fun 'space'/2), fun 'pair'/2]))), p_optional(fun 'space'/2), p_string(<<"}">>)]), p_seq([p_string(<<"{">>), p_optional(fun 'space'/2), p_string(<<"}">>)])]))(I,D) end, fun(Node, Idx) ->
case length(Node) of
3 -> {struct, []};
_ ->
Head = proplists:get_value(head, Node),
Rest = [lists:nth(4, I) || I <- proplists:get_value(tail, Node)],
{struct, [Head|Rest]}
end
end).
'pair'(Input, Index) ->
p(Input, Index, 'pair', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), p_label('key', fun 'string'/2), p_optional(fun 'space'/2), p_string(<<":">>), p_optional(fun 'space'/2), p_label('value', fun 'json_value'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, Idx) -> {proplists:get_value(key, Node), proplists:get_value(value, Node)} end).
'array'(Input, Index) ->
p(Input, Index, 'array', fun(I,D) -> (p_choose([p_seq([p_string(<<"[">>), p_optional(fun 'space'/2), p_label('head', fun 'json_value'/2), p_label('tail', p_zero_or_more(p_seq([p_optional(fun 'space'/2), p_string(<<",">>), p_optional(fun 'space'/2), fun 'json_value'/2]))), p_optional(fun 'space'/2), p_string(<<"]">>)]), p_seq([p_string(<<"[">>), p_optional(fun 'space'/2), p_string(<<"]">>)])]))(I,D) end, fun(Node, Idx) ->
case length(Node) of
3 -> [];
_ ->
Head = proplists:get_value(head, Node),
Rest = [lists:nth(4, I) || I <- proplists:get_value(tail, Node)],
[Head|Rest]
end
end).
'string'(Input, Index) ->
p(Input, Index, 'string', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('chars', p_zero_or_more(p_seq([p_not(p_string(<<"\"">>)), p_choose([p_string(<<"\\\\\\\\">>), p_string(<<"\\\\\"">>), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, Idx) -> unicode:characters_to_binary(proplists:get_value(chars, Node)) end).
'number'(Input, Index) ->
p(Input, Index, 'number', fun(I,D) -> (p_seq([fun 'int'/2, p_optional(fun 'frac'/2), p_optional(fun 'exp'/2)]))(I,D) end, fun(Node, Idx) ->
case Node of
[Int, [], []] -> list_to_integer(binary_to_list(unicode:characters_to_binary(Int)));
[Int, Frac, []] -> list_to_float(binary_to_list(unicode:characters_to_binary([Int, Frac])));
[Int, [], Exp] -> list_to_float(binary_to_list(unicode:characters_to_binary([Int, ".0", Exp])));
_ -> list_to_float(binary_to_list(unicode:characters_to_binary(Node)))
end
end).
'int'(Input, Index) ->
p(Input, Index, 'int', fun(I,D) -> (p_choose([p_seq([p_optional(p_string(<<"-">>)), p_seq([fun 'non_zero_digit'/2, p_one_or_more(fun 'digit'/2)])]), fun 'digit'/2]))(I,D) end, fun(Node, Idx) -> Node end).
'frac'(Input, Index) ->
p(Input, Index, 'frac', fun(I,D) -> (p_seq([p_string(<<".">>), p_one_or_more(fun 'digit'/2)]))(I,D) end, fun(Node, Idx) -> Node end).
'exp'(Input, Index) ->
p(Input, Index, 'exp', fun(I,D) -> (p_seq([fun 'e'/2, p_one_or_more(fun 'digit'/2)]))(I,D) end, fun(Node, Idx) -> Node end).
'e'(Input, Index) ->
p(Input, Index, 'e', fun(I,D) -> (p_seq([p_charclass(<<"[eE]">>), p_optional(p_choose([p_string(<<"+">>), p_string(<<"-">>)]))]))(I,D) end, fun(Node, Idx) -> Node end).
'non_zero_digit'(Input, Index) ->
p(Input, Index, 'non_zero_digit', fun(I,D) -> (p_charclass(<<"[1-9]">>))(I,D) end, fun(Node, Idx) -> Node end).
'digit'(Input, Index) ->
p(Input, Index, 'digit', fun(I,D) -> (p_charclass(<<"[0-9]">>))(I,D) end, fun(Node, Idx) -> Node end).
'true'(Input, Index) ->
p(Input, Index, 'true', fun(I,D) -> (p_string(<<"true">>))(I,D) end, fun(Node, Idx) -> true end).
'false'(Input, Index) ->
p(Input, Index, 'false', fun(I,D) -> (p_string(<<"false">>))(I,D) end, fun(Node, Idx) -> false end).
'null'(Input, Index) ->
p(Input, Index, 'null', fun(I,D) -> (p_string(<<"null">>))(I,D) end, fun(Node, Idx) -> null end).
'space'(Input, Index) ->
p(Input, Index, 'space', fun(I,D) -> (p_zero_or_more(p_charclass(<<"[ \t\n\s\r]">>)))(I,D) end, fun(Node, Idx) -> Node end).
p(Inp, Index, Name, ParseFun) ->
p(Inp, Index, Name, ParseFun, fun(N, _Idx) -> N end).
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
% Grab the memo table from ets
Memo = get_memo(StartIndex),
% See if the current reduction is memoized
case proplists:lookup(Name, Memo) of
% If it is, return the result
{Name, Result} -> Result;
% If not, attempt to parse
_ ->
case ParseFun(Inp, StartIndex) of
% If it fails, memoize the failure
{fail,_} = Failure ->
memoize(StartIndex, [{Name, Failure}|Memo]),
Failure;
% If it passes, transform and memoize the result.
{Result, InpRem, NewIndex} ->
Transformed = TransformFun(Result, StartIndex),
memoize(StartIndex, [{Name, {Transformed, InpRem, NewIndex}}|Memo]),
{Transformed, InpRem, NewIndex}
end
end.
setup_memo() ->
put(parse_memo_table, ets:new(?MODULE, [set])).
release_memo() ->
ets:delete(memo_table_name()).
memoize(Position, Struct) ->
ets:insert(memo_table_name(), {Position, Struct}).
get_memo(Position) ->
case ets:lookup(memo_table_name(), Position) of
[] -> [];
[{Position, PList}] -> PList
end.
memo_table_name() ->
get(parse_memo_table).
p_eof() ->
fun(<<>>, Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
p_and(P) ->
p_seq(P).
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
p_scan(_, [], Index, Accum) -> {lists:reverse( Accum ), [], Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
p_string(S) when is_list(S) -> p_string(list_to_binary(S));
p_string(S) ->
Length = erlang:byte_size(S),
fun(Input, Index) ->
try
<> = Input,
{S, Rest, p_advance_index(S, Index)}
catch
error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
end
end.
p_anything() ->
fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
(Input, Index) when is_binary(Input) ->
<> = Input,
{<>, Rest, p_advance_index(<>, Index)}
end.
p_charclass(Class) ->
{ok, RE} = re:compile(Class, [unicode, dotall]),
fun(Inp, Index) ->
case re:run(Inp, RE, [anchored]) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {character_class, Class}, Index}}
end
end.
line({{line,L},_}) -> L;
line(_) -> undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/extra/json.peg 0000664 0000000 0000000 00000002772 13113476405 0016711 0 ustar 00root root 0000000 0000000 json_value <- space? (object / array / string / number / true / false / null) space? `lists:nth(2, Node)`;
object <- '{' space? head:pair tail:(space? ',' space? pair)* space? '}' / '{' space? '}'
`
case length(Node) of
3 -> {struct, []};
_ ->
Head = proplists:get_value(head, Node),
Rest = [lists:nth(4, I) || I <- proplists:get_value(tail, Node)],
{struct, [Head|Rest]}
end
`;
pair <- space? key:string space? ':' space? value:json_value space? `{proplists:get_value(key, Node), proplists:get_value(value, Node)}`;
array <- '[' space? head:json_value tail:(space? ',' space? json_value)* space? ']' / '[' space? ']'
`
case length(Node) of
3 -> [];
_ ->
Head = proplists:get_value(head, Node),
Rest = [lists:nth(4, I) || I <- proplists:get_value(tail, Node)],
[Head|Rest]
end
`;
string <- '"' chars:(!'"' ("\\\\" / '\\"' / .))* '"' `iolist_to_binary(proplists:get_value(chars, Node))`;
number <- int frac? exp?
`
case Node of
[Int, [], []] -> list_to_integer(binary_to_list(iolist_to_binary(Int)));
[Int, Frac, []] -> list_to_float(binary_to_list(iolist_to_binary([Int, Frac])));
[Int, [], Exp] -> list_to_float(binary_to_list(iolist_to_binary([Int, ".0", Exp])));
_ -> list_to_float(binary_to_list(iolist_to_binary(Node)))
end
`;
int <- '-'? (non_zero_digit digit+) / digit ~;
frac <- '.' digit+ ~;
exp <- e digit+ ~;
e <- [eE] ('+' / '-')? ~;
non_zero_digit <- [1-9] ~;
digit <- [0-9] ~;
true <- 'true' `true`;
false <- 'false' `false`;
null <- 'null' `null`;
space <- [ \t\n\s\r]* ~; neotoma-1.7.4+dfsg/priv/ 0000775 0000000 0000000 00000000000 13113476405 0015070 5 ustar 00root root 0000000 0000000 neotoma-1.7.4+dfsg/priv/neotoma_parse.peg 0000664 0000000 0000000 00000021654 13113476405 0020431 0 ustar 00root root 0000000 0000000 %% Grammar and transformations
rules <- space? declaration_sequence space? code_block? space?
`
RootRule = verify_rules(),
Rules = unicode:characters_to_binary(lists:map(fun(R) -> [R, "\n\n"] end, lists:nth(2, Node))),
Code = case lists:nth(4, Node) of
{code, Block} -> Block;
_ -> []
end,
[{rules, Rules},
{code, Code},
{root, RootRule},
{transform, ets:lookup(memo_table_name(),gen_transform)},
{combinators, ets:lookup_element(memo_table_name(), combinators, 2)}]
`;
declaration_sequence <- head:declaration tail:(space declaration)*
`
FirstRule = proplists:get_value(head, Node),
OtherRules = [I || [_,I] <- proplists:get_value(tail, Node, [])],
[FirstRule|OtherRules]
`;
declaration <- nonterminal space* '<-' space* parsing_expression space? code_block? space? ';'
`
[{nonterminal,Symbol}|Tail] = Node,
add_lhs(Symbol, Index),
Transform = case lists:nth(6,Tail) of
{code, CodeBlock} -> CodeBlock;
_ ->
ets:insert_new(memo_table_name(),{gen_transform, true}),
["transform('",Symbol,"', Node, Idx)"]
end,
TransformArgs = case used_transform_variables(Transform) of
[] -> "_Node, _Idx";
['Idx'] -> "_Node, Idx";
['Node'] -> "Node, _Idx";
['Idx', 'Node'] -> "Node, Idx"
end,
["-spec '", Symbol, "'(input(), index()) -> parse_result().\n",
"'",Symbol,"'","(Input, Index) ->\n ",
"p(Input, Index, '",Symbol,"', fun(I,D) -> (",
lists:nth(4, Tail),
")(I,D) end, fun(", TransformArgs, ") ->",Transform," end)."]
`;
parsing_expression <- choice / sequence / primary ~;
choice <- head:alternative tail:(space '/' space alternative)+
`
Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_choose),
["p_choose([", Head, Statements, "])"]
`;
alternative <- sequence / labeled_primary ~;
primary <- prefix atomic / atomic suffix / atomic
`
case Node of
[Atomic, one_or_more] ->
used_combinator(p_one_or_more),
used_combinator(p_scan),
["p_one_or_more(", Atomic, ")"];
[Atomic, zero_or_more] ->
used_combinator(p_zero_or_more),
used_combinator(p_scan),
["p_zero_or_more(", Atomic, ")"];
[Atomic, optional] ->
used_combinator(p_optional),
["p_optional(", Atomic, ")"];
[assert, Atomic] ->
used_combinator(p_assert),
["p_assert(", Atomic, ")"];
[not_, Atomic] ->
used_combinator(p_not),
["p_not(", Atomic, ")"];
_ -> Node
end
`;
sequence <- head:labeled_primary tail:(space labeled_primary)+
`
Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_seq),
["p_seq([", Head, Statements, "])"]
`;
labeled_primary <- label? primary
`
case hd(Node) of
[] -> lists:nth(2, Node);
Label ->
used_combinator(p_label),
["p_label('", Label, "', ", lists:nth(2, Node), ")"]
end
`;
label <- alpha_char alphanumeric_char* ':'
`
lists:sublist(Node, length(Node)-1)
`;
suffix <- repetition_suffix / optional_suffix
`
case Node of
<<"*">> -> zero_or_more;
<<"+">> -> one_or_more;
<<"?">> -> optional
end
`;
optional_suffix <- '?' ~;
repetition_suffix <- '+' / '*' ~;
prefix <- '&' / '!'
`
case Node of
<<"&">> -> assert;
<<"!">> -> not_
end
`;
atomic <- terminal / nonterminal / parenthesized_expression
`
case Node of
{nonterminal, Symbol} ->
[<<"fun '">>, Symbol, <<"'/2">>];
_ -> Node
end
`;
parenthesized_expression <- '(' space? parsing_expression space? ')' `lists:nth(3, Node)`;
nonterminal <- alpha_char alphanumeric_char*
`
Symbol = unicode:characters_to_binary(Node),
add_nt(Symbol, Idx),
{nonterminal, Symbol}
`;
terminal <- regexp_string / quoted_string / character_class / anything_symbol ~;
regexp_string <- '#' string:(!'#' ('\\#' / .))+ '#'
`
used_combinator(p_regexp),
["p_regexp(<<\"",
% Escape \ and " as they are used in erlang string. Other sumbol stay as is.
% \ -> \\
% " -> \"
re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
"\">>)"]
`;
quoted_string <- single_quoted_string / double_quoted_string
`
used_combinator(p_string),
lists:flatten(["p_string(<<\"",
escape_string(unicode:characters_to_list(proplists:get_value(string, Node))),
"\">>)"])
`;
double_quoted_string <- '"' string:(!'"' ("\\\\" / '\\"' / .))* '"' ~;
single_quoted_string <- "'" string:(!"'" ("\\\\" / "\\'" / .))* "'" ~;
character_class <- '[' characters:(!']' ('\\\\' . / !'\\\\' .))+ ']'
`
used_combinator(p_charclass),
["p_charclass(<<\"[",
escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))),
"]\">>)"]
`;
anything_symbol <- '.' ` used_combinator(p_anything), <<"p_anything()">> `;
alpha_char <- [A-Za-z_] ~;
alphanumeric_char <- alpha_char / [0-9] ~;
space <- (white / comment_to_eol)+ ~;
comment_to_eol <- !'%{' '%' (!"\n" .)* ~;
white <- [ \t\n\r] ~;
code_block <- ( '%{' code:('\\%' / '$%' / !'%}' .)+ '%}' ) /
('`' code:('\\`' / '$`' / !'`' .)+ '`') /
'~'
`
case Node of
<<"~">> -> {code, <<"Node">>};
_ -> {code, proplists:get_value('code', Node)}
end
`;
%% Extra functions
`
% insert escapes into a string
-spec escape_string(string()) -> string().
escape_string(String) -> escape_string(String, []).
-spec escape_string(string(), string()) -> string().
escape_string([], Output) ->
lists:reverse(Output);
escape_string([H|T], Output) ->
escape_string(T,
case H of
$/ -> [$/,$\\|Output];
$\" -> [$\",$\\|Output]; % " comment inserted to help some editors with highlighting the generated parser
$\' -> [$\',$\\|Output]; % ' comment inserted to help some editors with highlighting the generated parser
$\b -> [$b,$\\|Output];
$\d -> [$d,$\\|Output];
$\e -> [$e,$\\|Output];
$\f -> [$f,$\\|Output];
$\n -> [$n,$\\|Output];
$\r -> [$r,$\\|Output];
$\s -> [$s,$\\|Output];
$\t -> [$t,$\\|Output];
$\v -> [$v,$\\|Output];
_ -> [H|Output]
end).
-spec add_lhs(binary(), index()) -> true.
add_lhs(Symbol, Index) ->
case ets:lookup(memo_table_name(), lhs) of
[] ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}]});
[{lhs, L}] when is_list(L) ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}|L]})
end.
-spec add_nt(binary(), index()) -> true | ok.
add_nt(Symbol, Index) ->
case ets:lookup(memo_table_name(), nts) of
[] ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}]});
[{nts, L}] when is_list(L) ->
case proplists:is_defined(Symbol, L) of
true ->
ok;
_ ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}|L]})
end
end.
-spec verify_rules() -> ok | no_return().
verify_rules() ->
[{lhs, LHS}] = ets:lookup(memo_table_name(), lhs),
[{nts, NTs}] = ets:lookup(memo_table_name(), nts),
[Root|NonRoots] = lists:reverse(LHS),
lists:foreach(fun({Sym,Idx}) ->
case proplists:is_defined(Sym, NTs) of
true ->
ok;
_ ->
io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx])
end
end, NonRoots),
lists:foreach(fun({S,I}) ->
case proplists:is_defined(S, LHS) of
true ->
ok;
_ ->
io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]),
exit({neotoma, {no_reduction, list_to_atom(binary_to_list(S))}})
end
end, NTs),
Root.
-spec used_combinator(atom()) -> true.
used_combinator(C) ->
case ets:lookup(memo_table_name(), combinators) of
[] ->
ets:insert(memo_table_name(), {combinators, ordsets:from_list([C])});
[{combinators, Cs}] ->
ets:insert(memo_table_name(), {combinators, ordsets:add_element(C, Cs)})
end.
-spec used_transform_variables(binary()) -> [ 'Node' | 'Idx' ].
used_transform_variables(Transform) ->
Code = unicode:characters_to_list(Transform),
{ok, Tokens, _} = erl_scan:string(Code),
used_transform_variables(Tokens, []).
used_transform_variables([{var, _, Name}|Tokens], Acc) ->
used_transform_variables(Tokens, case Name of
'Node' -> [Name | Acc];
'Idx' -> [Name | Acc];
_ -> Acc
end);
used_transform_variables([_|Tokens], Acc) ->
used_transform_variables(Tokens, Acc);
used_transform_variables([], Acc) ->
lists:usort(Acc).
`
neotoma-1.7.4+dfsg/priv/peg_includes.hrl 0000664 0000000 0000000 00000017124 13113476405 0020245 0 ustar 00root root 0000000 0000000 -file("peg_includes.hrl", 1).
-type index() :: {{line, pos_integer()}, {column, pos_integer()}}.
-type input() :: binary().
-type parse_failure() :: {fail, term()}.
-type parse_success() :: {term(), input(), index()}.
-type parse_result() :: parse_failure() | parse_success().
-type parse_fun() :: fun((input(), index()) -> parse_result()).
-type xform_fun() :: fun((input(), index()) -> term()).
-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result().
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
case get_memo(StartIndex, Name) of % See if the current reduction is memoized
{ok, Memo} -> %Memo; % If it is, return the stored result
Memo;
_ -> % If not, attempt to parse
Result = case ParseFun(Inp, StartIndex) of
{fail,_} = Failure -> % If it fails, memoize the failure
Failure;
{Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result.
Transformed = TransformFun(Match, StartIndex),
{Transformed, InpRem, NewIndex}
end,
memoize(StartIndex, Name, Result),
Result
end.
-spec setup_memo() -> ets:tid().
setup_memo() ->
put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])).
-spec release_memo() -> true.
release_memo() ->
ets:delete(memo_table_name()).
-spec memoize(index(), atom(), parse_result()) -> true.
memoize(Index, Name, Result) ->
Memo = case ets:lookup(memo_table_name(), Index) of
[] -> [];
[{Index, Plist}] -> Plist
end,
ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}).
-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}.
get_memo(Index, Name) ->
case ets:lookup(memo_table_name(), Index) of
[] -> {error, not_found};
[{Index, Plist}] ->
case proplists:lookup(Name, Plist) of
{Name, Result} -> {ok, Result};
_ -> {error, not_found}
end
end.
-spec memo_table_name() -> ets:tid().
memo_table_name() ->
get({parse_memo_table, ?MODULE}).
-ifdef(p_eof).
-spec p_eof() -> parse_fun().
p_eof() ->
fun(<<>>, Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
-endif.
-ifdef(p_optional).
-spec p_optional(parse_fun()) -> parse_fun().
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
-endif.
-ifdef(p_not).
-spec p_not(parse_fun()) -> parse_fun().
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
-endif.
-ifdef(p_assert).
-spec p_assert(parse_fun()) -> parse_fun().
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
-endif.
-ifdef(p_seq).
-spec p_seq([parse_fun()]) -> parse_fun().
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result().
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
-endif.
-ifdef(p_choose).
-spec p_choose([parse_fun()]) -> parse_fun().
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result().
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
-endif.
-ifdef(p_zero_or_more).
-spec p_zero_or_more(parse_fun()) -> parse_fun().
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
-endif.
-ifdef(p_one_or_more).
-spec p_one_or_more(parse_fun()) -> parse_fun().
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
-endif.
-ifdef(p_label).
-spec p_label(atom(), parse_fun()) -> parse_fun().
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
-endif.
-ifdef(p_scan).
-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}.
p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
-endif.
-ifdef(p_string).
-spec p_string(binary()) -> parse_fun().
p_string(S) ->
Length = erlang:byte_size(S),
fun(Input, Index) ->
try
<> = Input,
{S, Rest, p_advance_index(S, Index)}
catch
error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
end
end.
-endif.
-ifdef(p_anything).
-spec p_anything() -> parse_fun().
p_anything() ->
fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
(Input, Index) when is_binary(Input) ->
<> = Input,
{<>, Rest, p_advance_index(<>, Index)}
end.
-endif.
-ifdef(p_charclass).
-spec p_charclass(string() | binary()) -> parse_fun().
p_charclass(Class) ->
{ok, RE} = re:compile(Class, [unicode, dotall]),
fun(Inp, Index) ->
case re:run(Inp, RE, [anchored]) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}}
end
end.
-endif.
-ifdef(p_regexp).
-spec p_regexp(binary()) -> parse_fun().
p_regexp(Regexp) ->
{ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]),
fun(Inp, Index) ->
case re:run(Inp, RE) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}}
end
end.
-endif.
-ifdef(line).
-spec line(index() | term()) -> pos_integer() | undefined.
line({{line,L},_}) -> L;
line(_) -> undefined.
-endif.
-ifdef(column).
-spec column(index() | term()) -> pos_integer() | undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
-endif.
-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index().
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/rebar.config 0000664 0000000 0000000 00000000212 13113476405 0016365 0 ustar 00root root 0000000 0000000 {erl_opts, [debug_info, warn_export_vars]}.
{xref_checks, [undefined_function_calls]}.
{eunit_opts, [verbose]}.
{plugins, [rebar3_hex]}.
neotoma-1.7.4+dfsg/src/ 0000775 0000000 0000000 00000000000 13113476405 0014677 5 ustar 00root root 0000000 0000000 neotoma-1.7.4+dfsg/src/neotoma.app.src 0000664 0000000 0000000 00000000443 13113476405 0017632 0 ustar 00root root 0000000 0000000 {application, neotoma,
[
{description, "PEG/Packrat toolkit and parser-generator."},
{vsn, "1.7.4"},
{registered, []},
{applications, [kernel, stdlib]},
{contributors, ["Sean Cribbs"]},
{licenses, ["MIT"]},
{links, [{"Github", "https://github.com/seancribbs/neotoma"}]}
]
}.
neotoma-1.7.4+dfsg/src/neotoma.erl 0000664 0000000 0000000 00000015400 13113476405 0017045 0 ustar 00root root 0000000 0000000 -module(neotoma).
-author("Sean Cribbs ").
-export([file/1, file/2, bootstrap/0]).
-export([main/1]).
-define(ALL_COMBINATORS, [p_eof, p_optional, p_not, p_assert, p_seq,
p_choose, p_zero_or_more, p_one_or_more, p_label, p_scan,
p_string, p_anything, p_charclass, p_regexp, line, column]).
-type option() :: {module, atom()} | {output, file:filename()} | {transform_module, atom()} |
{neotoma_priv_dir, file:filename()}.
%% @doc Handler function for escript.
-spec main(list()) -> ok | no_return().
main([]) ->
io:format("Usage: neotoma filename [-module output_module] [-output output_dir] [-transform_module transform_module]\n");
main([Filename | Args]) ->
%% code:priv_dir is unreliable when called in escript context, but
%% escript:script_name does what we want.
PrivDir = filename:join([filename:dirname(escript:script_name()), "priv"]),
file(Filename, [{neotoma_priv_dir, PrivDir} | parse_options(Args)]).
%% @doc Generates a parser from the specified file.
%% @equiv file(Filename, [])
-spec file(file:filename()) -> ok | {error, atom()}.
file(InputGrammar) ->
file(InputGrammar, []).
%% @doc Generates a parser from the specified file with the given options.
-spec file(file:filename(), [option()]) -> ok | {error, atom()}.
file(InputGrammar, Options) ->
Basename = filename:basename(InputGrammar, ".peg"),
InputDir = filename:dirname(InputGrammar),
ModuleName = proplists:get_value(module, Options, list_to_atom(Basename)),
OutputDir = proplists:get_value(output, Options, InputDir),
OutputFilename = filename:join(OutputDir, atom_to_list(ModuleName) ++ ".erl"),
TransformModule = proplists:get_value(transform_module, Options, false),
validate_params(filename:absname(InputGrammar),
ModuleName,
TransformModule,
filename:absname(OutputFilename)),
Parsed = parse_grammar(InputGrammar),
Rules = proplists:get_value(rules, Parsed),
Root = proplists:get_value(root, Parsed),
Code = proplists:get_value(code, Parsed),
GenTransform = proplists:get_value(transform, Parsed),
Combinators = proplists:get_value(combinators, Parsed, ?ALL_COMBINATORS),
ModuleAttrs = generate_module_attrs(ModuleName, Combinators),
EntryFuns = generate_entry_functions(Root),
TransformFun = create_transform(TransformModule, OutputDir, GenTransform),
PrivDir = proplists:get_value(neotoma_priv_dir, Options, code:priv_dir(neotoma)),
{ok, PegIncludes} = file:read_file(filename:join([PrivDir, "peg_includes.hrl"])),
file:write_file(OutputFilename, [ModuleAttrs, "\n", Code, "\n", EntryFuns, "\n", Rules, "\n", TransformFun, "\n", PegIncludes]).
-spec validate_params(file:filename(),atom(),atom(),file:filename()) -> 'ok'.
validate_params(InputGrammar, _, _, OutputFile) when InputGrammar =:= OutputFile ->
throw({badarg, "Input and output file are the same!"});
validate_params(_,_, false, _) -> ok;
validate_params(_,_, TransformModule, _) when not is_atom(TransformModule) ->
throw({badarg, "transform_module option must be an atom"});
validate_params(_,Basename, TransformModule, _) when Basename =:= TransformModule ->
throw({badarg, "Transform module named same as parser module!"});
validate_params(_,_, TransformModule, OutputFile) ->
OutMod = list_to_atom(filename:basename(OutputFile, ".erl")),
case OutMod of
TransformModule -> throw({badarg, "Transform module file same as parser output file!"});
_ -> ok
end.
-spec generate_module_attrs(atom(), [atom()]) -> iolist().
generate_module_attrs(ModName, Combinators) ->
["-module(", atom_to_list(ModName) ,").\n",
"-export([parse/1,file/1]).\n",
[ generate_combinator_macro(C) || Combinators /= undefined, C <- Combinators ],
"\n"
].
generate_combinator_macro(C) ->
["-define(", atom_to_list(C), ",true).\n"].
-spec generate_entry_functions({iodata(),_}) -> iolist().
generate_entry_functions(Root) ->
{RootRule,_} = Root,
["-spec file(file:name()) -> any().\n",
"file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.\n\n",
"-spec parse(binary() | list()) -> any().\n",
"parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));\n",
"parse(Input) when is_binary(Input) ->\n",
" _ = setup_memo(),\n",
" Result = case '",RootRule,"'(Input,{{line,1},{column,1}}) of\n",
" {AST, <<>>, _Index} -> AST;\n",
" Any -> Any\n"
" end,\n",
" release_memo(), Result.\n"].
-spec parse_grammar(file:filename()) -> any().
parse_grammar(InputFile) ->
case neotoma_parse:file(InputFile) of
{fail, Index} ->
throw({grammar_error, {fail, Index}});
{Parsed, Remainder, Index} ->
io:format("WARNING: Grammar parse ended unexpectedly at ~p, generated parser may be incorrect.~nRemainder:~n~p",
[Index, Remainder]),
Parsed;
L when is_list(L) -> L;
_ -> throw({error, {unknown, grammar, InputFile}})
end.
-spec create_transform(atom() | boolean(),file:filename(),_) -> iolist().
create_transform(_,_,[]) -> [];
create_transform(false,_,_) ->
"transform(_,Node,_Index) -> Node.";
create_transform(ModName,Dir,_) when is_atom(ModName) ->
XfFile = filename:join(Dir, atom_to_list(ModName) ++ ".erl"),
case filelib:is_regular(XfFile) of
true -> io:format("'~s' already exists, skipping generation.~n", [XfFile]);
false -> generate_transform_stub(XfFile, ModName)
end,
["transform(Symbol,Node,Index) -> ",atom_to_list(ModName),":transform(Symbol, Node, Index)."].
-spec generate_transform_stub(file:filename(), atom()) -> 'ok' | {'error',atom()}.
generate_transform_stub(XfFile,ModName) ->
Data = ["-module(",atom_to_list(ModName),").\n",
"-export([transform/3]).\n\n",
"%% Add clauses to this function to transform syntax nodes\n",
"%% from the parser into semantic output.\n",
"transform(Symbol, Node, _Index) when is_atom(Symbol) ->\n Node."],
file:write_file(XfFile, Data).
%% @doc Bootstraps the neotoma metagrammar. Intended only for internal development!
%% @equiv file("src/neotoma_parse.peg")
-spec bootstrap() -> 'ok'.
bootstrap() ->
file("priv/neotoma_parse.peg", [{output, "src/"}, {neotoma_priv_dir, "priv"}]).
%% @doc Parses arguments passed to escript
-spec parse_options(list()) -> list().
parse_options(["-module", ModName | Rest]) ->
[{module, list_to_atom(ModName)} | parse_options(Rest)];
parse_options(["-output", Dir | Rest]) ->
[{output, Dir} | parse_options(Rest)];
parse_options(["-transform_module", ModName | Rest]) ->
[{transform_module, list_to_atom(ModName)} | parse_options(Rest)];
parse_options([]) ->
[].
neotoma-1.7.4+dfsg/src/neotoma_parse.erl 0000664 0000000 0000000 00000057635 13113476405 0020257 0 ustar 00root root 0000000 0000000 -module(neotoma_parse).
-export([parse/1,file/1]).
-define(p_anything,true).
-define(p_charclass,true).
-define(p_choose,true).
-define(p_label,true).
-define(p_not,true).
-define(p_one_or_more,true).
-define(p_optional,true).
-define(p_scan,true).
-define(p_seq,true).
-define(p_string,true).
-define(p_zero_or_more,true).
% insert escapes into a string
-spec escape_string(string()) -> string().
escape_string(String) -> escape_string(String, []).
-spec escape_string(string(), string()) -> string().
escape_string([], Output) ->
lists:reverse(Output);
escape_string([H|T], Output) ->
escape_string(T,
case H of
$/ -> [$/,$\\|Output];
$\" -> [$\",$\\|Output]; % " comment inserted to help some editors with highlighting the generated parser
$\' -> [$\',$\\|Output]; % ' comment inserted to help some editors with highlighting the generated parser
$\b -> [$b,$\\|Output];
$\d -> [$d,$\\|Output];
$\e -> [$e,$\\|Output];
$\f -> [$f,$\\|Output];
$\n -> [$n,$\\|Output];
$\r -> [$r,$\\|Output];
$\s -> [$s,$\\|Output];
$\t -> [$t,$\\|Output];
$\v -> [$v,$\\|Output];
_ -> [H|Output]
end).
-spec add_lhs(binary(), index()) -> true.
add_lhs(Symbol, Index) ->
case ets:lookup(memo_table_name(), lhs) of
[] ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}]});
[{lhs, L}] when is_list(L) ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}|L]})
end.
-spec add_nt(binary(), index()) -> true | ok.
add_nt(Symbol, Index) ->
case ets:lookup(memo_table_name(), nts) of
[] ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}]});
[{nts, L}] when is_list(L) ->
case proplists:is_defined(Symbol, L) of
true ->
ok;
_ ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}|L]})
end
end.
-spec verify_rules() -> ok | no_return().
verify_rules() ->
[{lhs, LHS}] = ets:lookup(memo_table_name(), lhs),
[{nts, NTs}] = ets:lookup(memo_table_name(), nts),
[Root|NonRoots] = lists:reverse(LHS),
lists:foreach(fun({Sym,Idx}) ->
case proplists:is_defined(Sym, NTs) of
true ->
ok;
_ ->
io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx])
end
end, NonRoots),
lists:foreach(fun({S,I}) ->
case proplists:is_defined(S, LHS) of
true ->
ok;
_ ->
io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]),
exit({neotoma, {no_reduction, list_to_atom(binary_to_list(S))}})
end
end, NTs),
Root.
-spec used_combinator(atom()) -> true.
used_combinator(C) ->
case ets:lookup(memo_table_name(), combinators) of
[] ->
ets:insert(memo_table_name(), {combinators, ordsets:from_list([C])});
[{combinators, Cs}] ->
ets:insert(memo_table_name(), {combinators, ordsets:add_element(C, Cs)})
end.
-spec used_transform_variables(binary()) -> [ 'Node' | 'Idx' ].
used_transform_variables(Transform) ->
Code = unicode:characters_to_list(Transform),
{ok, Tokens, _} = erl_scan:string(Code),
used_transform_variables(Tokens, []).
used_transform_variables([{var, _, Name}|Tokens], Acc) ->
used_transform_variables(Tokens, case Name of
'Node' -> [Name | Acc];
'Idx' -> [Name | Acc];
_ -> Acc
end);
used_transform_variables([_|Tokens], Acc) ->
used_transform_variables(Tokens, Acc);
used_transform_variables([], Acc) ->
lists:usort(Acc).
-spec file(file:name()) -> any().
file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.
-spec parse(binary() | list()) -> any().
parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));
parse(Input) when is_binary(Input) ->
_ = setup_memo(),
Result = case 'rules'(Input,{{line,1},{column,1}}) of
{AST, <<>>, _Index} -> AST;
Any -> Any
end,
release_memo(), Result.
-spec 'rules'(input(), index()) -> parse_result().
'rules'(Input, Index) ->
p(Input, Index, 'rules', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), fun 'declaration_sequence'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, _Idx) ->
RootRule = verify_rules(),
Rules = unicode:characters_to_binary(lists:map(fun(R) -> [R, "\n\n"] end, lists:nth(2, Node))),
Code = case lists:nth(4, Node) of
{code, Block} -> Block;
_ -> []
end,
[{rules, Rules},
{code, Code},
{root, RootRule},
{transform, ets:lookup(memo_table_name(),gen_transform)},
{combinators, ets:lookup_element(memo_table_name(), combinators, 2)}]
end).
-spec 'declaration_sequence'(input(), index()) -> parse_result().
'declaration_sequence'(Input, Index) ->
p(Input, Index, 'declaration_sequence', fun(I,D) -> (p_seq([p_label('head', fun 'declaration'/2), p_label('tail', p_zero_or_more(p_seq([fun 'space'/2, fun 'declaration'/2])))]))(I,D) end, fun(Node, _Idx) ->
FirstRule = proplists:get_value(head, Node),
OtherRules = [I || [_,I] <- proplists:get_value(tail, Node, [])],
[FirstRule|OtherRules]
end).
-spec 'declaration'(input(), index()) -> parse_result().
'declaration'(Input, Index) ->
p(Input, Index, 'declaration', fun(I,D) -> (p_seq([fun 'nonterminal'/2, p_zero_or_more(fun 'space'/2), p_string(<<"<-">>), p_zero_or_more(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2), p_string(<<";">>)]))(I,D) end, fun(Node, _Idx) ->
[{nonterminal,Symbol}|Tail] = Node,
add_lhs(Symbol, Index),
Transform = case lists:nth(6,Tail) of
{code, CodeBlock} -> CodeBlock;
_ ->
ets:insert_new(memo_table_name(),{gen_transform, true}),
["transform('",Symbol,"', Node, Idx)"]
end,
TransformArgs = case used_transform_variables(Transform) of
[] -> "_Node, _Idx";
['Idx'] -> "_Node, Idx";
['Node'] -> "Node, _Idx";
['Idx', 'Node'] -> "Node, Idx"
end,
["-spec '", Symbol, "'(input(), index()) -> parse_result().\n",
"'",Symbol,"'","(Input, Index) ->\n ",
"p(Input, Index, '",Symbol,"', fun(I,D) -> (",
lists:nth(4, Tail),
")(I,D) end, fun(", TransformArgs, ") ->",Transform," end)."]
end).
-spec 'parsing_expression'(input(), index()) -> parse_result().
'parsing_expression'(Input, Index) ->
p(Input, Index, 'parsing_expression', fun(I,D) -> (p_choose([fun 'choice'/2, fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'choice'(input(), index()) -> parse_result().
'choice'(Input, Index) ->
p(Input, Index, 'choice', fun(I,D) -> (p_seq([p_label('head', fun 'alternative'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, p_string(<<"\/">>), fun 'space'/2, fun 'alternative'/2])))]))(I,D) end, fun(Node, _Idx) ->
Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_choose),
["p_choose([", Head, Statements, "])"]
end).
-spec 'alternative'(input(), index()) -> parse_result().
'alternative'(Input, Index) ->
p(Input, Index, 'alternative', fun(I,D) -> (p_choose([fun 'sequence'/2, fun 'labeled_primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'primary'(input(), index()) -> parse_result().
'primary'(Input, Index) ->
p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([fun 'prefix'/2, fun 'atomic'/2]), p_seq([fun 'atomic'/2, fun 'suffix'/2]), fun 'atomic'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
[Atomic, one_or_more] ->
used_combinator(p_one_or_more),
used_combinator(p_scan),
["p_one_or_more(", Atomic, ")"];
[Atomic, zero_or_more] ->
used_combinator(p_zero_or_more),
used_combinator(p_scan),
["p_zero_or_more(", Atomic, ")"];
[Atomic, optional] ->
used_combinator(p_optional),
["p_optional(", Atomic, ")"];
[assert, Atomic] ->
used_combinator(p_assert),
["p_assert(", Atomic, ")"];
[not_, Atomic] ->
used_combinator(p_not),
["p_not(", Atomic, ")"];
_ -> Node
end
end).
-spec 'sequence'(input(), index()) -> parse_result().
'sequence'(Input, Index) ->
p(Input, Index, 'sequence', fun(I,D) -> (p_seq([p_label('head', fun 'labeled_primary'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, fun 'labeled_primary'/2])))]))(I,D) end, fun(Node, _Idx) ->
Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_seq),
["p_seq([", Head, Statements, "])"]
end).
-spec 'labeled_primary'(input(), index()) -> parse_result().
'labeled_primary'(Input, Index) ->
p(Input, Index, 'labeled_primary', fun(I,D) -> (p_seq([p_optional(fun 'label'/2), fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->
case hd(Node) of
[] -> lists:nth(2, Node);
Label ->
used_combinator(p_label),
["p_label('", Label, "', ", lists:nth(2, Node), ")"]
end
end).
-spec 'label'(input(), index()) -> parse_result().
'label'(Input, Index) ->
p(Input, Index, 'label', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2), p_string(<<":">>)]))(I,D) end, fun(Node, _Idx) ->
lists:sublist(Node, length(Node)-1)
end).
-spec 'suffix'(input(), index()) -> parse_result().
'suffix'(Input, Index) ->
p(Input, Index, 'suffix', fun(I,D) -> (p_choose([fun 'repetition_suffix'/2, fun 'optional_suffix'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"*">> -> zero_or_more;
<<"+">> -> one_or_more;
<<"?">> -> optional
end
end).
-spec 'optional_suffix'(input(), index()) -> parse_result().
'optional_suffix'(Input, Index) ->
p(Input, Index, 'optional_suffix', fun(I,D) -> (p_string(<<"?">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'repetition_suffix'(input(), index()) -> parse_result().
'repetition_suffix'(Input, Index) ->
p(Input, Index, 'repetition_suffix', fun(I,D) -> (p_choose([p_string(<<"+">>), p_string(<<"*">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'prefix'(input(), index()) -> parse_result().
'prefix'(Input, Index) ->
p(Input, Index, 'prefix', fun(I,D) -> (p_choose([p_string(<<"&">>), p_string(<<"!">>)]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"&">> -> assert;
<<"!">> -> not_
end
end).
-spec 'atomic'(input(), index()) -> parse_result().
'atomic'(Input, Index) ->
p(Input, Index, 'atomic', fun(I,D) -> (p_choose([fun 'terminal'/2, fun 'nonterminal'/2, fun 'parenthesized_expression'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
{nonterminal, Symbol} ->
[<<"fun '">>, Symbol, <<"'/2">>];
_ -> Node
end
end).
-spec 'parenthesized_expression'(input(), index()) -> parse_result().
'parenthesized_expression'(Input, Index) ->
p(Input, Index, 'parenthesized_expression', fun(I,D) -> (p_seq([p_string(<<"(">>), p_optional(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(<<")">>)]))(I,D) end, fun(Node, _Idx) ->lists:nth(3, Node) end).
-spec 'nonterminal'(input(), index()) -> parse_result().
'nonterminal'(Input, Index) ->
p(Input, Index, 'nonterminal', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2)]))(I,D) end, fun(Node, Idx) ->
Symbol = unicode:characters_to_binary(Node),
add_nt(Symbol, Idx),
{nonterminal, Symbol}
end).
-spec 'terminal'(input(), index()) -> parse_result().
'terminal'(Input, Index) ->
p(Input, Index, 'terminal', fun(I,D) -> (p_choose([fun 'regexp_string'/2, fun 'quoted_string'/2, fun 'character_class'/2, fun 'anything_symbol'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'regexp_string'(input(), index()) -> parse_result().
'regexp_string'(Input, Index) ->
p(Input, Index, 'regexp_string', fun(I,D) -> (p_seq([p_string(<<"#">>), p_label('string', p_one_or_more(p_seq([p_not(p_string(<<"#">>)), p_choose([p_string(<<"\\#">>), p_anything()])]))), p_string(<<"#">>)]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_regexp),
["p_regexp(<<\"",
% Escape \ and " as they are used in erlang string. Other sumbol stay as is.
% \ -> \\
% " -> \"
re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
"\">>)"]
end).
-spec 'quoted_string'(input(), index()) -> parse_result().
'quoted_string'(Input, Index) ->
p(Input, Index, 'quoted_string', fun(I,D) -> (p_choose([fun 'single_quoted_string'/2, fun 'double_quoted_string'/2]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_string),
lists:flatten(["p_string(<<\"",
escape_string(unicode:characters_to_list(proplists:get_value(string, Node))),
"\">>)"])
end).
-spec 'double_quoted_string'(input(), index()) -> parse_result().
'double_quoted_string'(Input, Index) ->
p(Input, Index, 'double_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\"">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\"">>), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'single_quoted_string'(input(), index()) -> parse_result().
'single_quoted_string'(Input, Index) ->
p(Input, Index, 'single_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\'">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\'">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\'">>), p_anything()])]))), p_string(<<"\'">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'character_class'(input(), index()) -> parse_result().
'character_class'(Input, Index) ->
p(Input, Index, 'character_class', fun(I,D) -> (p_seq([p_string(<<"[">>), p_label('characters', p_one_or_more(p_seq([p_not(p_string(<<"]">>)), p_choose([p_seq([p_string(<<"\\\\">>), p_anything()]), p_seq([p_not(p_string(<<"\\\\">>)), p_anything()])])]))), p_string(<<"]">>)]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_charclass),
["p_charclass(<<\"[",
escape_string(unicode:characters_to_list(proplists:get_value(characters, Node))),
"]\">>)"]
end).
-spec 'anything_symbol'(input(), index()) -> parse_result().
'anything_symbol'(Input, Index) ->
p(Input, Index, 'anything_symbol', fun(I,D) -> (p_string(<<".">>))(I,D) end, fun(_Node, _Idx) -> used_combinator(p_anything), <<"p_anything()">> end).
-spec 'alpha_char'(input(), index()) -> parse_result().
'alpha_char'(Input, Index) ->
p(Input, Index, 'alpha_char', fun(I,D) -> (p_charclass(<<"[A-Za-z_]">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'alphanumeric_char'(input(), index()) -> parse_result().
'alphanumeric_char'(Input, Index) ->
p(Input, Index, 'alphanumeric_char', fun(I,D) -> (p_choose([fun 'alpha_char'/2, p_charclass(<<"[0-9]">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'space'(input(), index()) -> parse_result().
'space'(Input, Index) ->
p(Input, Index, 'space', fun(I,D) -> (p_one_or_more(p_choose([fun 'white'/2, fun 'comment_to_eol'/2])))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'comment_to_eol'(input(), index()) -> parse_result().
'comment_to_eol'(Input, Index) ->
p(Input, Index, 'comment_to_eol', fun(I,D) -> (p_seq([p_not(p_string(<<"%{">>)), p_string(<<"%">>), p_zero_or_more(p_seq([p_not(p_string(<<"\n">>)), p_anything()]))]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'white'(input(), index()) -> parse_result().
'white'(Input, Index) ->
p(Input, Index, 'white', fun(I,D) -> (p_charclass(<<"[\s\t\n\r]">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'code_block'(input(), index()) -> parse_result().
'code_block'(Input, Index) ->
p(Input, Index, 'code_block', fun(I,D) -> (p_choose([p_seq([p_string(<<"%{">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\%">>), p_string(<<"$%">>), p_seq([p_not(p_string(<<"%}">>)), p_anything()])]))), p_string(<<"%}">>)]), p_seq([p_string(<<"`">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\`">>), p_string(<<"$`">>), p_seq([p_not(p_string(<<"`">>)), p_anything()])]))), p_string(<<"`">>)]), p_string(<<"~">>)]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"~">> -> {code, <<"Node">>};
_ -> {code, proplists:get_value('code', Node)}
end
end).
-file("peg_includes.hrl", 1).
-type index() :: {{line, pos_integer()}, {column, pos_integer()}}.
-type input() :: binary().
-type parse_failure() :: {fail, term()}.
-type parse_success() :: {term(), input(), index()}.
-type parse_result() :: parse_failure() | parse_success().
-type parse_fun() :: fun((input(), index()) -> parse_result()).
-type xform_fun() :: fun((input(), index()) -> term()).
-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result().
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
case get_memo(StartIndex, Name) of % See if the current reduction is memoized
{ok, Memo} -> %Memo; % If it is, return the stored result
Memo;
_ -> % If not, attempt to parse
Result = case ParseFun(Inp, StartIndex) of
{fail,_} = Failure -> % If it fails, memoize the failure
Failure;
{Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result.
Transformed = TransformFun(Match, StartIndex),
{Transformed, InpRem, NewIndex}
end,
memoize(StartIndex, Name, Result),
Result
end.
-spec setup_memo() -> ets:tid().
setup_memo() ->
put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])).
-spec release_memo() -> true.
release_memo() ->
ets:delete(memo_table_name()).
-spec memoize(index(), atom(), parse_result()) -> true.
memoize(Index, Name, Result) ->
Memo = case ets:lookup(memo_table_name(), Index) of
[] -> [];
[{Index, Plist}] -> Plist
end,
ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}).
-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}.
get_memo(Index, Name) ->
case ets:lookup(memo_table_name(), Index) of
[] -> {error, not_found};
[{Index, Plist}] ->
case proplists:lookup(Name, Plist) of
{Name, Result} -> {ok, Result};
_ -> {error, not_found}
end
end.
-spec memo_table_name() -> ets:tid().
memo_table_name() ->
get({parse_memo_table, ?MODULE}).
-ifdef(p_eof).
-spec p_eof() -> parse_fun().
p_eof() ->
fun(<<>>, Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
-endif.
-ifdef(p_optional).
-spec p_optional(parse_fun()) -> parse_fun().
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
-endif.
-ifdef(p_not).
-spec p_not(parse_fun()) -> parse_fun().
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
-endif.
-ifdef(p_assert).
-spec p_assert(parse_fun()) -> parse_fun().
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
-endif.
-ifdef(p_seq).
-spec p_seq([parse_fun()]) -> parse_fun().
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result().
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
-endif.
-ifdef(p_choose).
-spec p_choose([parse_fun()]) -> parse_fun().
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result().
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
-endif.
-ifdef(p_zero_or_more).
-spec p_zero_or_more(parse_fun()) -> parse_fun().
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
-endif.
-ifdef(p_one_or_more).
-spec p_one_or_more(parse_fun()) -> parse_fun().
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
-endif.
-ifdef(p_label).
-spec p_label(atom(), parse_fun()) -> parse_fun().
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
-endif.
-ifdef(p_scan).
-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}.
p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
-endif.
-ifdef(p_string).
-spec p_string(binary()) -> parse_fun().
p_string(S) ->
Length = erlang:byte_size(S),
fun(Input, Index) ->
try
<> = Input,
{S, Rest, p_advance_index(S, Index)}
catch
error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
end
end.
-endif.
-ifdef(p_anything).
-spec p_anything() -> parse_fun().
p_anything() ->
fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
(Input, Index) when is_binary(Input) ->
<> = Input,
{<>, Rest, p_advance_index(<>, Index)}
end.
-endif.
-ifdef(p_charclass).
-spec p_charclass(string() | binary()) -> parse_fun().
p_charclass(Class) ->
{ok, RE} = re:compile(Class, [unicode, dotall]),
fun(Inp, Index) ->
case re:run(Inp, RE, [anchored]) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}}
end
end.
-endif.
-ifdef(p_regexp).
-spec p_regexp(binary()) -> parse_fun().
p_regexp(Regexp) ->
{ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]),
fun(Inp, Index) ->
case re:run(Inp, RE) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}}
end
end.
-endif.
-ifdef(line).
-spec line(index() | term()) -> pos_integer() | undefined.
line({{line,L},_}) -> L;
line(_) -> undefined.
-endif.
-ifdef(column).
-spec column(index() | term()) -> pos_integer() | undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
-endif.
-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index().
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/src/neotoma_parse.erl.safe 0000664 0000000 0000000 00000057615 13113476405 0021172 0 ustar 00root root 0000000 0000000 -module(neotoma_parse).
-export([parse/1,file/1]).
-define(p_anything,true).
-define(p_charclass,true).
-define(p_choose,true).
-define(p_label,true).
-define(p_not,true).
-define(p_one_or_more,true).
-define(p_optional,true).
-define(p_scan,true).
-define(p_seq,true).
-define(p_string,true).
-define(p_zero_or_more,true).
% insert escapes into a string
-spec escape_string(string()) -> string().
escape_string(String) -> escape_string(String, []).
-spec escape_string(string(), string()) -> string().
escape_string([], Output) ->
lists:reverse(Output);
escape_string([H|T], Output) ->
escape_string(T,
case H of
$/ -> [$/,$\\|Output];
$\" -> [$\",$\\|Output]; % " comment inserted to help some editors with highlighting the generated parser
$\' -> [$\',$\\|Output]; % ' comment inserted to help some editors with highlighting the generated parser
$\b -> [$b,$\\|Output];
$\d -> [$d,$\\|Output];
$\e -> [$e,$\\|Output];
$\f -> [$f,$\\|Output];
$\n -> [$n,$\\|Output];
$\r -> [$r,$\\|Output];
$\s -> [$s,$\\|Output];
$\t -> [$t,$\\|Output];
$\v -> [$v,$\\|Output];
_ -> [H|Output]
end).
-spec add_lhs(binary(), index()) -> true.
add_lhs(Symbol, Index) ->
case ets:lookup(memo_table_name(), lhs) of
[] ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}]});
[{lhs, L}] when is_list(L) ->
ets:insert(memo_table_name(), {lhs, [{Symbol,Index}|L]})
end.
-spec add_nt(binary(), index()) -> true | ok.
add_nt(Symbol, Index) ->
case ets:lookup(memo_table_name(), nts) of
[] ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}]});
[{nts, L}] when is_list(L) ->
case proplists:is_defined(Symbol, L) of
true ->
ok;
_ ->
ets:insert(memo_table_name(), {nts, [{Symbol,Index}|L]})
end
end.
-spec verify_rules() -> ok | no_return().
verify_rules() ->
[{lhs, LHS}] = ets:lookup(memo_table_name(), lhs),
[{nts, NTs}] = ets:lookup(memo_table_name(), nts),
[Root|NonRoots] = lists:reverse(LHS),
lists:foreach(fun({Sym,Idx}) ->
case proplists:is_defined(Sym, NTs) of
true ->
ok;
_ ->
io:format("neotoma warning: rule '~s' is unused. ~p~n", [Sym,Idx])
end
end, NonRoots),
lists:foreach(fun({S,I}) ->
case proplists:is_defined(S, LHS) of
true ->
ok;
_ ->
io:format("neotoma error: nonterminal '~s' has no reduction. (found at ~p) No parser will be generated!~n", [S,I]),
exit({neotoma, {no_reduction, list_to_atom(binary_to_list(S))}})
end
end, NTs),
Root.
-spec used_combinator(atom()) -> true.
used_combinator(C) ->
case ets:lookup(memo_table_name(), combinators) of
[] ->
ets:insert(memo_table_name(), {combinators, ordsets:from_list([C])});
[{combinators, Cs}] ->
ets:insert(memo_table_name(), {combinators, ordsets:add_element(C, Cs)})
end.
-spec used_transform_variables(binary()) -> [ 'Node' | 'Idx' ].
used_transform_variables(Transform) ->
Code = unicode:characters_to_list(Transform),
{ok, Tokens, _} = erl_scan:string(Code),
used_transform_variables(Tokens, []).
used_transform_variables([{var, _, Name}|Tokens], Acc) ->
used_transform_variables(Tokens, case Name of
'Node' -> [Name | Acc];
'Idx' -> [Name | Acc];
_ -> Acc
end);
used_transform_variables([_|Tokens], Acc) ->
used_transform_variables(Tokens, Acc);
used_transform_variables([], Acc) ->
lists:usort(Acc).
-spec file(file:name()) -> any().
file(Filename) -> case file:read_file(Filename) of {ok,Bin} -> parse(Bin); Err -> Err end.
-spec parse(binary() | list()) -> any().
parse(List) when is_list(List) -> parse(unicode:characters_to_binary(List));
parse(Input) when is_binary(Input) ->
setup_memo(),
Result = case 'rules'(Input,{{line,1},{column,1}}) of
{AST, <<>>, _Index} -> AST;
Any -> Any
end,
release_memo(), Result.
-spec 'rules'(input(), index()) -> parse_result().
'rules'(Input, Index) ->
p(Input, Index, 'rules', fun(I,D) -> (p_seq([p_optional(fun 'space'/2), fun 'declaration_sequence'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2)]))(I,D) end, fun(Node, _Idx) ->
RootRule = verify_rules(),
Rules = iolist_to_binary(lists:map(fun(R) -> [R, "\n\n"] end, lists:nth(2, Node))),
Code = case lists:nth(4, Node) of
{code, Block} -> Block;
_ -> []
end,
[{rules, Rules},
{code, Code},
{root, RootRule},
{transform, ets:lookup(memo_table_name(),gen_transform)},
{combinators, ets:lookup_element(memo_table_name(), combinators, 2)}]
end).
-spec 'declaration_sequence'(input(), index()) -> parse_result().
'declaration_sequence'(Input, Index) ->
p(Input, Index, 'declaration_sequence', fun(I,D) -> (p_seq([p_label('head', fun 'declaration'/2), p_label('tail', p_zero_or_more(p_seq([fun 'space'/2, fun 'declaration'/2])))]))(I,D) end, fun(Node, _Idx) ->
FirstRule = proplists:get_value(head, Node),
OtherRules = [I || [_,I] <- proplists:get_value(tail, Node, [])],
[FirstRule|OtherRules]
end).
-spec 'declaration'(input(), index()) -> parse_result().
'declaration'(Input, Index) ->
p(Input, Index, 'declaration', fun(I,D) -> (p_seq([fun 'nonterminal'/2, p_zero_or_more(fun 'space'/2), p_string(<<"<-">>), p_zero_or_more(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_optional(fun 'code_block'/2), p_optional(fun 'space'/2), p_string(<<";">>)]))(I,D) end, fun(Node, _Idx) ->
[{nonterminal,Symbol}|Tail] = Node,
add_lhs(Symbol, Index),
Transform = case lists:nth(6,Tail) of
{code, CodeBlock} -> CodeBlock;
_ ->
ets:insert_new(memo_table_name(),{gen_transform, true}),
["transform('",Symbol,"', Node, Idx)"]
end,
TransformArgs = case used_transform_variables(Transform) of
[] -> "_Node, _Idx";
['Idx'] -> "_Node, Idx";
['Node'] -> "Node, _Idx";
['Idx', 'Node'] -> "Node, Idx"
end,
["-spec '", Symbol, "'(input(), index()) -> parse_result().\n",
"'",Symbol,"'","(Input, Index) ->\n ",
"p(Input, Index, '",Symbol,"', fun(I,D) -> (",
lists:nth(4, Tail),
")(I,D) end, fun(", TransformArgs, ") ->",Transform," end)."]
end).
-spec 'parsing_expression'(input(), index()) -> parse_result().
'parsing_expression'(Input, Index) ->
p(Input, Index, 'parsing_expression', fun(I,D) -> (p_choose([fun 'choice'/2, fun 'sequence'/2, fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'choice'(input(), index()) -> parse_result().
'choice'(Input, Index) ->
p(Input, Index, 'choice', fun(I,D) -> (p_seq([p_label('head', fun 'alternative'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, p_string(<<"\/">>), fun 'space'/2, fun 'alternative'/2])))]))(I,D) end, fun(Node, _Idx) ->
Tail = [lists:last(S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_choose),
["p_choose([", Head, Statements, "])"]
end).
-spec 'alternative'(input(), index()) -> parse_result().
'alternative'(Input, Index) ->
p(Input, Index, 'alternative', fun(I,D) -> (p_choose([fun 'sequence'/2, fun 'labeled_primary'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'primary'(input(), index()) -> parse_result().
'primary'(Input, Index) ->
p(Input, Index, 'primary', fun(I,D) -> (p_choose([p_seq([fun 'prefix'/2, fun 'atomic'/2]), p_seq([fun 'atomic'/2, fun 'suffix'/2]), fun 'atomic'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
[Atomic, one_or_more] ->
used_combinator(p_one_or_more),
used_combinator(p_scan),
["p_one_or_more(", Atomic, ")"];
[Atomic, zero_or_more] ->
used_combinator(p_zero_or_more),
used_combinator(p_scan),
["p_zero_or_more(", Atomic, ")"];
[Atomic, optional] ->
used_combinator(p_optional),
["p_optional(", Atomic, ")"];
[assert, Atomic] ->
used_combinator(p_assert),
["p_assert(", Atomic, ")"];
[not_, Atomic] ->
used_combinator(p_not),
["p_not(", Atomic, ")"];
_ -> Node
end
end).
-spec 'sequence'(input(), index()) -> parse_result().
'sequence'(Input, Index) ->
p(Input, Index, 'sequence', fun(I,D) -> (p_seq([p_label('head', fun 'labeled_primary'/2), p_label('tail', p_one_or_more(p_seq([fun 'space'/2, fun 'labeled_primary'/2])))]))(I,D) end, fun(Node, _Idx) ->
Tail = [lists:nth(2, S) || S <- proplists:get_value(tail, Node)],
Head = proplists:get_value(head, Node),
Statements = [[", ", TS] || TS <- Tail],
used_combinator(p_seq),
["p_seq([", Head, Statements, "])"]
end).
-spec 'labeled_primary'(input(), index()) -> parse_result().
'labeled_primary'(Input, Index) ->
p(Input, Index, 'labeled_primary', fun(I,D) -> (p_seq([p_optional(fun 'label'/2), fun 'primary'/2]))(I,D) end, fun(Node, _Idx) ->
case hd(Node) of
[] -> lists:nth(2, Node);
Label ->
used_combinator(p_label),
["p_label('", Label, "', ", lists:nth(2, Node), ")"]
end
end).
-spec 'label'(input(), index()) -> parse_result().
'label'(Input, Index) ->
p(Input, Index, 'label', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2), p_string(<<":">>)]))(I,D) end, fun(Node, _Idx) ->
lists:sublist(Node, length(Node)-1)
end).
-spec 'suffix'(input(), index()) -> parse_result().
'suffix'(Input, Index) ->
p(Input, Index, 'suffix', fun(I,D) -> (p_choose([fun 'repetition_suffix'/2, fun 'optional_suffix'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"*">> -> zero_or_more;
<<"+">> -> one_or_more;
<<"?">> -> optional
end
end).
-spec 'optional_suffix'(input(), index()) -> parse_result().
'optional_suffix'(Input, Index) ->
p(Input, Index, 'optional_suffix', fun(I,D) -> (p_string(<<"?">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'repetition_suffix'(input(), index()) -> parse_result().
'repetition_suffix'(Input, Index) ->
p(Input, Index, 'repetition_suffix', fun(I,D) -> (p_choose([p_string(<<"+">>), p_string(<<"*">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'prefix'(input(), index()) -> parse_result().
'prefix'(Input, Index) ->
p(Input, Index, 'prefix', fun(I,D) -> (p_choose([p_string(<<"&">>), p_string(<<"!">>)]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"&">> -> assert;
<<"!">> -> not_
end
end).
-spec 'atomic'(input(), index()) -> parse_result().
'atomic'(Input, Index) ->
p(Input, Index, 'atomic', fun(I,D) -> (p_choose([fun 'terminal'/2, fun 'nonterminal'/2, fun 'parenthesized_expression'/2]))(I,D) end, fun(Node, _Idx) ->
case Node of
{nonterminal, Symbol} ->
[<<"fun '">>, Symbol, <<"'/2">>];
_ -> Node
end
end).
-spec 'parenthesized_expression'(input(), index()) -> parse_result().
'parenthesized_expression'(Input, Index) ->
p(Input, Index, 'parenthesized_expression', fun(I,D) -> (p_seq([p_string(<<"(">>), p_optional(fun 'space'/2), fun 'parsing_expression'/2, p_optional(fun 'space'/2), p_string(<<")">>)]))(I,D) end, fun(Node, _Idx) ->lists:nth(3, Node) end).
-spec 'nonterminal'(input(), index()) -> parse_result().
'nonterminal'(Input, Index) ->
p(Input, Index, 'nonterminal', fun(I,D) -> (p_seq([fun 'alpha_char'/2, p_zero_or_more(fun 'alphanumeric_char'/2)]))(I,D) end, fun(Node, Idx) ->
Symbol = iolist_to_binary(Node),
add_nt(Symbol, Idx),
{nonterminal, Symbol}
end).
-spec 'terminal'(input(), index()) -> parse_result().
'terminal'(Input, Index) ->
p(Input, Index, 'terminal', fun(I,D) -> (p_choose([fun 'regexp_string'/2, fun 'quoted_string'/2, fun 'character_class'/2, fun 'anything_symbol'/2]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'regexp_string'(input(), index()) -> parse_result().
'regexp_string'(Input, Index) ->
p(Input, Index, 'regexp_string', fun(I,D) -> (p_seq([p_string(<<"#">>), p_label('string', p_one_or_more(p_seq([p_not(p_string(<<"#">>)), p_choose([p_string(<<"\\#">>), p_anything()])]))), p_string(<<"#">>)]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_regexp),
["p_regexp(<<\"",
% Escape \ and " as they are used in erlang string. Other sumbol stay as is.
% \ -> \\
% " -> \"
re:replace(proplists:get_value(string, Node), "\"|\\\\", "\\\\&", [{return, binary}, global]),
"\">>)"]
end).
-spec 'quoted_string'(input(), index()) -> parse_result().
'quoted_string'(Input, Index) ->
p(Input, Index, 'quoted_string', fun(I,D) -> (p_choose([fun 'single_quoted_string'/2, fun 'double_quoted_string'/2]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_string),
lists:flatten(["p_string(<<\"",
escape_string(binary_to_list(iolist_to_binary(proplists:get_value(string, Node)))),
"\">>)"])
end).
-spec 'double_quoted_string'(input(), index()) -> parse_result().
'double_quoted_string'(Input, Index) ->
p(Input, Index, 'double_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\"">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\"">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\"">>), p_anything()])]))), p_string(<<"\"">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'single_quoted_string'(input(), index()) -> parse_result().
'single_quoted_string'(Input, Index) ->
p(Input, Index, 'single_quoted_string', fun(I,D) -> (p_seq([p_string(<<"\'">>), p_label('string', p_zero_or_more(p_seq([p_not(p_string(<<"\'">>)), p_choose([p_string(<<"\\\\">>), p_string(<<"\\\'">>), p_anything()])]))), p_string(<<"\'">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'character_class'(input(), index()) -> parse_result().
'character_class'(Input, Index) ->
p(Input, Index, 'character_class', fun(I,D) -> (p_seq([p_string(<<"[">>), p_label('characters', p_one_or_more(p_seq([p_not(p_string(<<"]">>)), p_choose([p_seq([p_string(<<"\\\\">>), p_anything()]), p_seq([p_not(p_string(<<"\\\\">>)), p_anything()])])]))), p_string(<<"]">>)]))(I,D) end, fun(Node, _Idx) ->
used_combinator(p_charclass),
["p_charclass(<<\"[",
escape_string(binary_to_list(iolist_to_binary(proplists:get_value(characters, Node)))),
"]\">>)"]
end).
-spec 'anything_symbol'(input(), index()) -> parse_result().
'anything_symbol'(Input, Index) ->
p(Input, Index, 'anything_symbol', fun(I,D) -> (p_string(<<".">>))(I,D) end, fun(_Node, _Idx) -> used_combinator(p_anything), <<"p_anything()">> end).
-spec 'alpha_char'(input(), index()) -> parse_result().
'alpha_char'(Input, Index) ->
p(Input, Index, 'alpha_char', fun(I,D) -> (p_charclass(<<"[A-Za-z_]">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'alphanumeric_char'(input(), index()) -> parse_result().
'alphanumeric_char'(Input, Index) ->
p(Input, Index, 'alphanumeric_char', fun(I,D) -> (p_choose([fun 'alpha_char'/2, p_charclass(<<"[0-9]">>)]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'space'(input(), index()) -> parse_result().
'space'(Input, Index) ->
p(Input, Index, 'space', fun(I,D) -> (p_one_or_more(p_choose([fun 'white'/2, fun 'comment_to_eol'/2])))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'comment_to_eol'(input(), index()) -> parse_result().
'comment_to_eol'(Input, Index) ->
p(Input, Index, 'comment_to_eol', fun(I,D) -> (p_seq([p_not(p_string(<<"%{">>)), p_string(<<"%">>), p_zero_or_more(p_seq([p_not(p_string(<<"\n">>)), p_anything()]))]))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'white'(input(), index()) -> parse_result().
'white'(Input, Index) ->
p(Input, Index, 'white', fun(I,D) -> (p_charclass(<<"[\s\t\n\r]">>))(I,D) end, fun(Node, _Idx) ->Node end).
-spec 'code_block'(input(), index()) -> parse_result().
'code_block'(Input, Index) ->
p(Input, Index, 'code_block', fun(I,D) -> (p_choose([p_seq([p_string(<<"%{">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\%">>), p_string(<<"$%">>), p_seq([p_not(p_string(<<"%}">>)), p_anything()])]))), p_string(<<"%}">>)]), p_seq([p_string(<<"`">>), p_label('code', p_one_or_more(p_choose([p_string(<<"\\`">>), p_string(<<"$`">>), p_seq([p_not(p_string(<<"`">>)), p_anything()])]))), p_string(<<"`">>)]), p_string(<<"~">>)]))(I,D) end, fun(Node, _Idx) ->
case Node of
<<"~">> -> {code, <<"Node">>};
_ -> {code, proplists:get_value('code', Node)}
end
end).
-file("peg_includes.hrl", 1).
-type index() :: {{line, pos_integer()}, {column, pos_integer()}}.
-type input() :: binary().
-type parse_failure() :: {fail, term()}.
-type parse_success() :: {term(), input(), index()}.
-type parse_result() :: parse_failure() | parse_success().
-type parse_fun() :: fun((input(), index()) -> parse_result()).
-type xform_fun() :: fun((input(), index()) -> term()).
-spec p(input(), index(), atom(), parse_fun(), xform_fun()) -> parse_result().
p(Inp, StartIndex, Name, ParseFun, TransformFun) ->
case get_memo(StartIndex, Name) of % See if the current reduction is memoized
{ok, Memo} -> %Memo; % If it is, return the stored result
Memo;
_ -> % If not, attempt to parse
Result = case ParseFun(Inp, StartIndex) of
{fail,_} = Failure -> % If it fails, memoize the failure
Failure;
{Match, InpRem, NewIndex} -> % If it passes, transform and memoize the result.
Transformed = TransformFun(Match, StartIndex),
{Transformed, InpRem, NewIndex}
end,
memoize(StartIndex, Name, Result),
Result
end.
-spec setup_memo() -> ets:tid().
setup_memo() ->
put({parse_memo_table, ?MODULE}, ets:new(?MODULE, [set])).
-spec release_memo() -> true.
release_memo() ->
ets:delete(memo_table_name()).
-spec memoize(index(), atom(), parse_result()) -> true.
memoize(Index, Name, Result) ->
Memo = case ets:lookup(memo_table_name(), Index) of
[] -> [];
[{Index, Plist}] -> Plist
end,
ets:insert(memo_table_name(), {Index, [{Name, Result}|Memo]}).
-spec get_memo(index(), atom()) -> {ok, term()} | {error, not_found}.
get_memo(Index, Name) ->
case ets:lookup(memo_table_name(), Index) of
[] -> {error, not_found};
[{Index, Plist}] ->
case proplists:lookup(Name, Plist) of
{Name, Result} -> {ok, Result};
_ -> {error, not_found}
end
end.
-spec memo_table_name() -> ets:tid().
memo_table_name() ->
get({parse_memo_table, ?MODULE}).
-ifdef(p_eof).
-spec p_eof() -> parse_fun().
p_eof() ->
fun(<<>>, Index) -> {eof, [], Index};
(_, Index) -> {fail, {expected, eof, Index}} end.
-endif.
-ifdef(p_optional).
-spec p_optional(parse_fun()) -> parse_fun().
p_optional(P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} -> {[], Input, Index};
{_, _, _} = Success -> Success
end
end.
-endif.
-ifdef(p_not).
-spec p_not(parse_fun()) -> parse_fun().
p_not(P) ->
fun(Input, Index)->
case P(Input,Index) of
{fail,_} ->
{[], Input, Index};
{Result, _, _} -> {fail, {expected, {no_match, Result},Index}}
end
end.
-endif.
-ifdef(p_assert).
-spec p_assert(parse_fun()) -> parse_fun().
p_assert(P) ->
fun(Input,Index) ->
case P(Input,Index) of
{fail,_} = Failure-> Failure;
_ -> {[], Input, Index}
end
end.
-endif.
-ifdef(p_seq).
-spec p_seq([parse_fun()]) -> parse_fun().
p_seq(P) ->
fun(Input, Index) ->
p_all(P, Input, Index, [])
end.
-spec p_all([parse_fun()], input(), index(), [term()]) -> parse_result().
p_all([], Inp, Index, Accum ) -> {lists:reverse( Accum ), Inp, Index};
p_all([P|Parsers], Inp, Index, Accum) ->
case P(Inp, Index) of
{fail, _} = Failure -> Failure;
{Result, InpRem, NewIndex} -> p_all(Parsers, InpRem, NewIndex, [Result|Accum])
end.
-endif.
-ifdef(p_choose).
-spec p_choose([parse_fun()]) -> parse_fun().
p_choose(Parsers) ->
fun(Input, Index) ->
p_attempt(Parsers, Input, Index, none)
end.
-spec p_attempt([parse_fun()], input(), index(), none | parse_failure()) -> parse_result().
p_attempt([], _Input, _Index, Failure) -> Failure;
p_attempt([P|Parsers], Input, Index, FirstFailure)->
case P(Input, Index) of
{fail, _} = Failure ->
case FirstFailure of
none -> p_attempt(Parsers, Input, Index, Failure);
_ -> p_attempt(Parsers, Input, Index, FirstFailure)
end;
Result -> Result
end.
-endif.
-ifdef(p_zero_or_more).
-spec p_zero_or_more(parse_fun()) -> parse_fun().
p_zero_or_more(P) ->
fun(Input, Index) ->
p_scan(P, Input, Index, [])
end.
-endif.
-ifdef(p_one_or_more).
-spec p_one_or_more(parse_fun()) -> parse_fun().
p_one_or_more(P) ->
fun(Input, Index)->
Result = p_scan(P, Input, Index, []),
case Result of
{[_|_], _, _} ->
Result;
_ ->
{fail, {expected, Failure, _}} = P(Input,Index),
{fail, {expected, {at_least_one, Failure}, Index}}
end
end.
-endif.
-ifdef(p_label).
-spec p_label(atom(), parse_fun()) -> parse_fun().
p_label(Tag, P) ->
fun(Input, Index) ->
case P(Input, Index) of
{fail,_} = Failure ->
Failure;
{Result, InpRem, NewIndex} ->
{{Tag, Result}, InpRem, NewIndex}
end
end.
-endif.
-ifdef(p_scan).
-spec p_scan(parse_fun(), input(), index(), [term()]) -> {[term()], input(), index()}.
p_scan(_, <<>>, Index, Accum) -> {lists:reverse(Accum), <<>>, Index};
p_scan(P, Inp, Index, Accum) ->
case P(Inp, Index) of
{fail,_} -> {lists:reverse(Accum), Inp, Index};
{Result, InpRem, NewIndex} -> p_scan(P, InpRem, NewIndex, [Result | Accum])
end.
-endif.
-ifdef(p_string).
-spec p_string(binary()) -> parse_fun().
p_string(S) ->
Length = erlang:byte_size(S),
fun(Input, Index) ->
try
<> = Input,
{S, Rest, p_advance_index(S, Index)}
catch
error:{badmatch,_} -> {fail, {expected, {string, S}, Index}}
end
end.
-endif.
-ifdef(p_anything).
-spec p_anything() -> parse_fun().
p_anything() ->
fun(<<>>, Index) -> {fail, {expected, any_character, Index}};
(Input, Index) when is_binary(Input) ->
<> = Input,
{<>, Rest, p_advance_index(<>, Index)}
end.
-endif.
-ifdef(p_charclass).
-spec p_charclass(string() | binary()) -> parse_fun().
p_charclass(Class) ->
{ok, RE} = re:compile(Class, [unicode, dotall]),
fun(Inp, Index) ->
case re:run(Inp, RE, [anchored]) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {character_class, binary_to_list(Class)}, Index}}
end
end.
-endif.
-ifdef(p_regexp).
-spec p_regexp(binary()) -> parse_fun().
p_regexp(Regexp) ->
{ok, RE} = re:compile(Regexp, [unicode, dotall, anchored]),
fun(Inp, Index) ->
case re:run(Inp, RE) of
{match, [{0, Length}|_]} ->
{Head, Tail} = erlang:split_binary(Inp, Length),
{Head, Tail, p_advance_index(Head, Index)};
_ -> {fail, {expected, {regexp, binary_to_list(Regexp)}, Index}}
end
end.
-endif.
-ifdef(line).
-spec line(index() | term()) -> pos_integer() | undefined.
line({{line,L},_}) -> L;
line(_) -> undefined.
-endif.
-ifdef(column).
-spec column(index() | term()) -> pos_integer() | undefined.
column({_,{column,C}}) -> C;
column(_) -> undefined.
-endif.
-spec p_advance_index(input() | unicode:charlist() | pos_integer(), index()) -> index().
p_advance_index(MatchedInput, Index) when is_list(MatchedInput) orelse is_binary(MatchedInput)-> % strings
lists:foldl(fun p_advance_index/2, Index, unicode:characters_to_list(MatchedInput));
p_advance_index(MatchedInput, Index) when is_integer(MatchedInput) -> % single characters
{{line, Line}, {column, Col}} = Index,
case MatchedInput of
$\n -> {{line, Line+1}, {column, 1}};
_ -> {{line, Line}, {column, Col+1}}
end.
neotoma-1.7.4+dfsg/test/ 0000775 0000000 0000000 00000000000 13113476405 0015067 5 ustar 00root root 0000000 0000000 neotoma-1.7.4+dfsg/test/neotoma_peg.erl 0000664 0000000 0000000 00000001725 13113476405 0020075 0 ustar 00root root 0000000 0000000 -module(neotoma_peg).
-author("Sean Cribbs ").
% Thanks to Jeffrey A. Meunier for the original parser.erl library from which I
% lifted many of these functions, which in turn was based on the Haskell
% "parsec" library by Erik Meijer. I've renamed the functions to be more
% Erlang-y.
-define(p_eof,true).
-define(p_optional,true).
-define(p_not,true).
-define(p_assert,true).
-define(p_seq,true).
-define(p_choose,true).
-define(p_zero_or_more,true).
-define(p_one_or_more,true).
-define(p_label,true).
-define(p_scan,true).
-define(p_string,true).
-define(p_anything,true).
-define(p_charclass,true).
-define(p_regexp,true).
-define(line,true).
-define(column,true).
-export([p/5]).
-export([setup_memo/0, release_memo/0]).
-export([p_eof/0, p_optional/1, p_not/1, p_assert/1, p_seq/1, p_choose/1, p_zero_or_more/1, p_one_or_more/1, p_label/2, p_string/1, p_anything/0, p_charclass/1, p_regexp/1, line/1, column/1]).
-include("priv/peg_includes.hrl").
neotoma-1.7.4+dfsg/test/test_combinators.erl 0000664 0000000 0000000 00000013570 13113476405 0021160 0 ustar 00root root 0000000 0000000 -module(test_combinators).
-author("Sean Cribbs ").
-include_lib("eunit/include/eunit.hrl").
% Test the parser-combinators in the 'neotoma_peg' module
-define(STARTINDEX, {{line,1},{column,1}}).
eof_test_() ->
[
?_assertEqual({fail,{expected,eof,?STARTINDEX}}, (neotoma_peg:p_eof())(<<"abc">>,?STARTINDEX)),
?_assertEqual({eof, [], ?STARTINDEX}, (neotoma_peg:p_eof())(<<>>,?STARTINDEX))
].
optional_test_() ->
[
?_assertEqual({[], <<"xyz">>,?STARTINDEX}, (neotoma_peg:p_optional(neotoma_peg:p_string(<<"abc">>)))(<<"xyz">>,?STARTINDEX)),
?_assertEqual({<<"abc">>, <<"xyz">>,{{line,1},{column,4}}}, (neotoma_peg:p_optional(neotoma_peg:p_string(<<"abc">>)))(<<"abcxyz">>,?STARTINDEX))
].
not_test_() ->
[
?_assertEqual({[], <<"xyzabc">>,?STARTINDEX}, (neotoma_peg:p_not(neotoma_peg:p_string(<<"abc">>)))(<<"xyzabc">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {no_match, <<"abc">>}, ?STARTINDEX}}, (neotoma_peg:p_not(neotoma_peg:p_string(<<"abc">>)))(<<"abcxyz">>,?STARTINDEX))
].
assert_test_() ->
[
?_assertEqual({fail,{expected, {string, <<"abc">>}, ?STARTINDEX}}, (neotoma_peg:p_assert(neotoma_peg:p_string(<<"abc">>)))(<<"xyzabc">>,?STARTINDEX)),
?_assertEqual({[], <<"abcxyz">>,?STARTINDEX}, (neotoma_peg:p_assert(neotoma_peg:p_string(<<"abc">>)))(<<"abcxyz">>,?STARTINDEX))
].
seq_test_() ->
[
?_assertEqual({[<<"abc">>,<<"def">>], <<"xyz">>,{{line,1},{column,7}}}, (neotoma_peg:p_seq([neotoma_peg:p_string(<<"abc">>), neotoma_peg:p_string(<<"def">>)]))(<<"abcdefxyz">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {string, <<"def">>}, {{line,1},{column,4}}}}, (neotoma_peg:p_seq([neotoma_peg:p_string(<<"abc">>), neotoma_peg:p_string(<<"def">>)]))(<<"abcxyz">>,?STARTINDEX))
].
choose_test_() ->
[
?_assertEqual({<<"abc">>, <<"xyz">>, {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string(<<"abc">>), neotoma_peg:p_string(<<"def">>)]))(<<"abcxyz">>,?STARTINDEX)),
?_assertEqual({<<"def">>, <<"xyz">>, {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string(<<"abc">>), neotoma_peg:p_string(<<"def">>)]))(<<"defxyz">>,?STARTINDEX)),
?_assertEqual({<<"xyz">>, <<"xyz">>, {{line,1},{column,4}}}, (neotoma_peg:p_choose([neotoma_peg:p_string(<<"abc">>), neotoma_peg:p_string(<<"def">>), neotoma_peg:p_string(<<"xyz">>)]))(<<"xyzxyz">>,?STARTINDEX)),
?_assertEqual({fail,{expected,{string,<<"abc">>},?STARTINDEX}}, (neotoma_peg:p_choose([neotoma_peg:p_string(<<"abc">>),neotoma_peg:p_string(<<"def">>)]))(<<"xyz">>, ?STARTINDEX))
].
zero_or_more_test_() ->
[
?_assertEqual({[], <<>>, ?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"">>,?STARTINDEX)),
?_assertEqual({[], <<"def">>,?STARTINDEX}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"def">>,?STARTINDEX)),
?_assertEqual({[<<"abc">>], <<"def">>,{{line,1},{column,4}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"abcdef">>,?STARTINDEX)),
?_assertEqual({[<<"abc">>, <<"abc">>], <<"def">>,{{line,1},{column,7}}}, (neotoma_peg:p_zero_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"abcabcdef">>,?STARTINDEX))
].
one_or_more_test_() ->
[
?_assertEqual({fail,{expected,
{at_least_one,
{string, <<"abc">>}
}, ?STARTINDEX}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"def">>,?STARTINDEX)),
?_assertEqual({[<<"abc">>], <<"def">>,{{line,1},{column,4}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"abcdef">>,?STARTINDEX)),
?_assertEqual({[<<"abc">>,<<"abc">>], <<"def">>,{{line,1},{column,7}}}, (neotoma_peg:p_one_or_more(neotoma_peg:p_string(<<"abc">>)))(<<"abcabcdef">>,?STARTINDEX))
].
label_test_() ->
[
?_assertEqual({fail,{expected, {string, <<"!">>}, ?STARTINDEX}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string(<<"!">>)))(<<"?">>,?STARTINDEX)),
?_assertEqual({{bang, <<"!">>}, <<"">>,{{line,1},{column,2}}}, (neotoma_peg:p_label(bang, neotoma_peg:p_string(<<"!">>)))(<<"!">>,?STARTINDEX))
].
string_test_() ->
[
?_assertEqual({<<"abc">>, <<"def">>,{{line,1},{column,4}}}, (neotoma_peg:p_string(<<"abc">>))(<<"abcdef">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {string, <<"abc">>}, ?STARTINDEX}}, (neotoma_peg:p_string(<<"abc">>))(<<"defabc">>,?STARTINDEX))
].
anything_test_() ->
[
?_assertEqual({<<"a">>,<<"bcde">>,{{line,1},{column,2}}}, (neotoma_peg:p_anything())(<<"abcde">>,?STARTINDEX)),
?_assertEqual({<<"\n">>,<<"bcde">>,{{line,2},{column,1}}}, (neotoma_peg:p_anything())(<<"\nbcde">>,?STARTINDEX)),
?_assertEqual({fail,{expected, any_character, ?STARTINDEX}}, (neotoma_peg:p_anything())(<<"">>,?STARTINDEX))
].
charclass_test_() ->
[
?_assertEqual({<<"+">>,<<"----">>,{{line,1},{column,2}}}, (neotoma_peg:p_charclass(<<"[+]">>))(<<"+----">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {character_class, "[+]"}, ?STARTINDEX}}, (neotoma_peg:p_charclass(<<"[+]">>))(<<"----">>,?STARTINDEX))
].
regexp_test_() ->
RegexpFun = neotoma_peg:p_regexp(<<"a*">>),
[
?_assertEqual({<<"">>,<<"bbb">>,{{line,1},{column,1}}}, RegexpFun(<<"bbb">>,?STARTINDEX)),
?_assertEqual({<<"a">>,<<"bbb">>,{{line,1},{column,2}}}, RegexpFun(<<"abbb">>,?STARTINDEX)),
?_assertEqual({<<"aaa">>,<<"bbb">>,{{line,1},{column,4}}}, RegexpFun(<<"aaabbb">>,?STARTINDEX))
].
line_test() ->
?assertEqual(1, neotoma_peg:line({{line,1},{column,2}})).
column_test() ->
?assertEqual(2, neotoma_peg:column({{line,1},{column,2}})).
utf8_string_test_() ->
[
?_assertEqual({<<"世界">>, <<"def">>,{{line,1},{column,3}}}, (neotoma_peg:p_string(<<"世界">>))(<<"世界def">>,?STARTINDEX)),
?_assertEqual({fail,{expected, {string, <<"世界">>}, ?STARTINDEX}}, (neotoma_peg:p_string(<<"世界">>))(<<"界世abc">>,?STARTINDEX))
].
neotoma-1.7.4+dfsg/test/test_memoization.erl 0000664 0000000 0000000 00000002406 13113476405 0021167 0 ustar 00root root 0000000 0000000 -module(test_memoization).
-author("Sean Cribbs ").
-include_lib("eunit/include/eunit.hrl").
-define(I, fun(V,_) -> V end).
setup_memo_test() ->
neotoma_peg:setup_memo(),
?assertNot(undefined == ets:info(get({parse_memo_table, neotoma_peg}))),
neotoma_peg:release_memo().
release_memo_test() ->
neotoma_peg:setup_memo(),
neotoma_peg:release_memo(),
?assertEqual(undefined, ets:info(get({parse_memo_table, neotoma_peg}))).
step_memo_test() ->
neotoma_peg:setup_memo(),
Result = neotoma_peg:p(<<"abcdefghi">>, {{line,1},{column,1}}, anything, neotoma_peg:p_anything(), ?I),
?assertEqual({<<"a">>, <<"bcdefghi">>, {{line,1},{column,2}}}, Result),
Result2 = neotoma_peg:p(<<"abcdefghi">>, {{line,1},{column,1}}, anything, fun(_) ->
throw(bork) end, ?I),
?assertEqual(Result, Result2),
neotoma_peg:release_memo().
concurrent_memo_test() ->
Me = self(),
Him = spawn(fun() -> Me ! neotoma_peg:setup_memo(), receive _ -> ok after 10000 -> ok end end),
MyTid = neotoma_peg:setup_memo(),
receive
Tid -> ?assertNot(Tid == MyTid),
Him ! ok
after 10000 -> ok
end,
neotoma_peg:release_memo().
neotoma-1.7.4+dfsg/test/test_parse.erl 0000664 0000000 0000000 00000001157 13113476405 0017750 0 ustar 00root root 0000000 0000000 -module(test_parse).
-include_lib("eunit/include/eunit.hrl").
parser_test() ->
% so we don't have to copy test.peg to .eunit
Data = "rule <- .+;",
file:write_file("test_parser.peg", io_lib:fwrite("~s\n", [Data])),
neotoma:file("test_parser.peg"),
compile:file("test_parser.erl", []),
try
TestString = [19990,30028,32,102,111,111],
Result = test_parser:parse(TestString),
?assertEqual(6, length(Result)),
StringResult = lists:flatten(io_lib:format("~ts", [Result])),
?assertEqual(TestString, StringResult)
catch
_:_ -> ?assert(false)
end.
neotoma-1.7.4+dfsg/test/test_suite.erl 0000664 0000000 0000000 00000000315 13113476405 0017762 0 ustar 00root root 0000000 0000000 -module(test_suite).
-author("Sean Cribbs ").
-include_lib("eunit/include/eunit.hrl").
all_test_() ->
[{module, test_combinators},{module, test_memoization},{module, test_parse}].