phply-1.2.4/0000755000175000017500000000000013253173557014423 5ustar viraptorviraptor00000000000000phply-1.2.4/tests/0000755000175000017500000000000013253173557015565 5ustar viraptorviraptor00000000000000phply-1.2.4/tests/__init__.py0000644000175000017500000000000013006525440017650 0ustar viraptorviraptor00000000000000phply-1.2.4/tests/test_lexer.py0000644000175000017500000003072313173612576020322 0ustar viraptorviraptor00000000000000from __future__ import print_function from phply import phplex import nose.tools import pprint def eq_tokens(input, expected, ignore=('WHITESPACE', 'OPEN_TAG', 'CLOSE_TAG')): output = [] lexer = phplex.full_lexer.clone() lexer.input(input) while True: tok = lexer.token() if not tok: break if tok.type in ignore: continue output.append((tok.type, tok.value)) print('Lexer output:') pprint.pprint(output) print() print('Token by token:') for out, exp in zip(output, expected): print('\tgot:', out, '\texpected:', exp) nose.tools.eq_(out, exp) assert len(output) == len(expected), \ 'output length was %d, expected %s' % (len(output), len(expected)) def test_whitespace(): input = ' \t\t ' expected = [ ('INLINE_HTML', ' '), ('OPEN_TAG', ''), ('INLINE_HTML', '\t\t '), ('OPEN_TAG', ''), ('INLINE_HTML', ' '), ('OPEN_TAG', ''), ] eq_tokens(input, expected, ignore=()) def test_open_close_tags(): input = ' <% %> <%= %>' expected = [ ('OPEN_TAG', ''), ('INLINE_HTML', ' '), ('OPEN_TAG', '<%'), ('WHITESPACE', ' '), ('CLOSE_TAG', '%>'), ('INLINE_HTML', ' '), ('OPEN_TAG', ''), ('INLINE_HTML', ' '), ('OPEN_TAG_WITH_ECHO', ''), ('INLINE_HTML', ' '), ('OPEN_TAG_WITH_ECHO', '<%='), ('WHITESPACE', ' '), ('CLOSE_TAG', '%>'), ] eq_tokens(input, expected, ignore=()) def test_numbers(): input = """""" expected = [ ('LNUMBER', '0'), ('LNUMBER', '12'), ('DNUMBER', '34.56'), ('DNUMBER', '7e8'), ('DNUMBER', '9.01e23'), ('DNUMBER', '4.5E+6'), ('DNUMBER', '.78e-9'), ('DNUMBER', '1.e+2'), ('DNUMBER', '34.'), ('DNUMBER', '.56'), ('LNUMBER', '0xdEcAfBaD'), ('LNUMBER', '0x123456789abcdef'), ('LNUMBER', '0666'), ] eq_tokens(input, expected) def test_strings(): input = r"""""" expected = [ ('CONSTANT_ENCAPSED_STRING', "''"), ('CONSTANT_ENCAPSED_STRING', "'hello'"), ('CONSTANT_ENCAPSED_STRING', "'what\\'s up'"), ('CONSTANT_ENCAPSED_STRING', "'newlines\n'"), ('QUOTE', '"'), ('QUOTE', '"'), ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', 'hello'), ('QUOTE', '"'), ('QUOTE', '"'), ('VARIABLE', '$world'), ('QUOTE', '"'), ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', 'hello '), ('VARIABLE', '$cruel'), ('ENCAPSED_AND_WHITESPACE', ' \\"world\\"'), ('QUOTE', '"'), ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', 'end$'), ('QUOTE', '"'), ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', 'newlines\n'), ('QUOTE', '"'), ] eq_tokens(input, expected) def test_string_backslash_escapes(): input = r"""""" expected = [ ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', "\n \\$escape\n \\{"), ('VARIABLE', "$escape"), ('ENCAPSED_AND_WHITESPACE', "}\n \\${escape}\n "), ('QUOTE', '"'), ] eq_tokens(input, expected) def test_string_offset_lookups(): input = r"""property $too->many->properties $adjacent->object$lookup stray -> [ ] not[array] non->object " ?>""" expected = [ ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', '\n '), ('VARIABLE', '$array'), ('LBRACKET', '['), ('STRING', 'offset'), ('RBRACKET', ']'), ('ENCAPSED_AND_WHITESPACE', '\n '), ('VARIABLE', '$too'), ('LBRACKET', '['), ('STRING', 'many'), ('RBRACKET', ']'), ('ENCAPSED_AND_WHITESPACE', '[offsets]\n '), ('VARIABLE', '$next'), ('LBRACKET', '['), ('STRING', 'to'), ('RBRACKET', ']'), ('VARIABLE', '$array'), ('ENCAPSED_AND_WHITESPACE', '\n '), ('DOLLAR_OPEN_CURLY_BRACES', '${'), ('STRING_VARNAME', 'curly'), ('LBRACKET', '['), ('CONSTANT_ENCAPSED_STRING', "'offset'"), ('RBRACKET', ']'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', '\n '), ('VARIABLE', '$object'), ('OBJECT_OPERATOR', '->'), ('STRING', 'property'), ('ENCAPSED_AND_WHITESPACE', '\n '), ('VARIABLE', '$too'), ('OBJECT_OPERATOR', '->'), ('STRING', 'many'), ('ENCAPSED_AND_WHITESPACE', '->properties\n '), ('VARIABLE', '$adjacent'), ('OBJECT_OPERATOR', '->'), ('STRING', 'object'), ('VARIABLE', '$lookup'), ('ENCAPSED_AND_WHITESPACE', '\n stray -> [ ]\n not[array]\n non->object\n '), ('QUOTE', '"'), ] eq_tokens(input, expected) def test_string_curly_dollar_expressions(): input = r"""items[4]->five} {${$nasty}} {${funcall()}} {${$object->method()}} {$object->$variable} {$object->$variable[1]} {${static_class::variable}} {${static_class::$variable}} " ?>""" expected = [ ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', "\n a"), ('DOLLAR_OPEN_CURLY_BRACES', "${"), ('STRING_VARNAME', "dollar_curly"), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "b\n c"), ('CURLY_OPEN', "{"), ('VARIABLE', "$curly_dollar"), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "d\n e"), ('DOLLAR_OPEN_CURLY_BRACES', "${"), ('VARIABLE', "$dollar_curly_dollar"), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "f\n "), ('CURLY_OPEN', "{"), ('VARIABLE', "$array"), ('LBRACKET', '['), ('LNUMBER', "0"), ('RBRACKET', ']'), ('LBRACKET', '['), ('LNUMBER', "1"), ('RBRACKET', ']'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('VARIABLE', "$array"), ('LBRACKET', '['), ('CONSTANT_ENCAPSED_STRING', "'two'"), ('RBRACKET', ']'), ('LBRACKET', '['), ('LNUMBER', "3"), ('RBRACKET', ']'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('VARIABLE', "$object"), ('OBJECT_OPERATOR', "->"), ('STRING', "items"), ('LBRACKET', '['), ('LNUMBER', "4"), ('RBRACKET', ']'), ('OBJECT_OPERATOR', "->"), ('STRING', "five"), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('DOLLAR', '$'), ('LBRACE', '{'), ('VARIABLE', "$nasty"), ('RBRACE', '}'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('DOLLAR', "$"), ('LBRACE', "{"), ('STRING', "funcall"), ('LPAREN', "("), ('RPAREN', ")"), ('RBRACE', '}'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('DOLLAR', "$"), ('LBRACE', "{"), ('VARIABLE', "$object"), ('OBJECT_OPERATOR', "->"), ('STRING', "method"), ('LPAREN', "("), ('RPAREN', ")"), ('RBRACE', '}'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('VARIABLE', "$object"), ('OBJECT_OPERATOR', "->"), ('VARIABLE', "$variable"), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('VARIABLE', "$object"), ('OBJECT_OPERATOR', "->"), ('VARIABLE', "$variable"), ('LBRACKET', '['), ('LNUMBER', "1"), ('RBRACKET', ']'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('DOLLAR', "$"), ('LBRACE', "{"), ('STRING', "static_class"), ('DOUBLE_COLON', "::"), ('STRING', "variable"), ('RBRACE', '}'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('CURLY_OPEN', "{"), ('DOLLAR', "$"), ('LBRACE', "{"), ('STRING', "static_class"), ('DOUBLE_COLON', "::"), ('VARIABLE', "$variable"), ('RBRACE', '}'), ('RBRACE', '}'), ('ENCAPSED_AND_WHITESPACE', "\n "), ('QUOTE', '"'), ] eq_tokens(input, expected) def test_heredoc(): input = r"""variables This is not the EOT; this is: EOT; ?>""" expected = [ ('OPEN_TAG', ''), ('STRING', 'variables'), ('ENCAPSED_AND_WHITESPACE', '\n'), ('ENCAPSED_AND_WHITESPACE', 'This'), ('ENCAPSED_AND_WHITESPACE', ' is not the EOT; this is:\n'), ('END_HEREDOC', 'EOT'), ('SEMI', ';'), ('WHITESPACE', '\n '), ('CLOSE_TAG', '?>'), ] eq_tokens(input, expected, ignore=()) def test_heredoc_backslash_newline(): input = r"""""" expected = [ ('OPEN_TAG', ''), ] eq_tokens(input, expected, ignore=()) def test_commented_close_tag(): input = '\n' expected = [ ('OPEN_TAG', '\n'), # PHP seems inconsistent regarding ('OPEN_TAG', ''), ] eq_tokens(input, expected, ignore=()) def test_punctuation(): input = '' expected = [ ('LPAREN', '('), ('LBRACKET', '['), ('LBRACE', '{'), ('RBRACE', '}'), ('RBRACKET', ']'), ('RPAREN', ')'), ('COLON', ':'), ('SEMI', ';'), ('COMMA', ','), ('CONCAT', '.'), ('AT', '@'), ] eq_tokens(input, expected) def test_backticks(): input = '' expected = [ ('BACKTICK', '`'), ('ENCAPSED_AND_WHITESPACE', 'ls '), ('VARIABLE', '$one'), ('BACKTICK', '`'), ('SEMI', ';'), ('QUOTE', '"'), ('ENCAPSED_AND_WHITESPACE', '`o``ne`'), ('QUOTE', '"'), ('SEMI', ';'), ('CONSTANT_ENCAPSED_STRING', "'`one`'"), ('SEMI', ';'), ] eq_tokens(input, expected) def test_exceptions(): input = ' more html' expected = [InlineHTML('html '), InlineHTML(' more html')] eq_ast(input, expected) def test_echo(): input = '' expected = [Echo(["hello, world!"])] eq_ast(input, expected) def test_open_tag_with_echo(): input = '' expected = [ Echo(["hello, world!"]), Echo(["test"]), Constant('EXTRA'), ] eq_ast(input, expected) def test_exit(): input = '' expected = [ Exit(None, 'exit'), Exit(None, 'exit'), Exit(123, 'exit'), Exit(None, 'die'), Exit(None, 'die'), Exit(456, 'die'), ] eq_ast(input, expected) def test_isset(): input = r"""c); isset($d['e']); isset($f, $g); isset($h->m()['i1']['i2']); ?>""" expected = [ IsSet([Variable('$a')]), IsSet([ObjectProperty(Variable('$b'), 'c')]), IsSet([ArrayOffset(Variable('$d'), 'e')]), IsSet([Variable('$f'), Variable('$g')]), IsSet([ArrayOffset(ArrayOffset(MethodCall(Variable('$h'), 'm', []), 'i1'), 'i2')]), ] eq_ast(input, expected) def test_namespace_names(): input = r"""""" expected = [ Constant(r'foo'), Constant(r'bar\baz'), Constant(r'one\too\tree'), Constant(r'\top'), Constant(r'\top\level'), Constant(r'namespace\level'), ] eq_ast(input, expected) def test_unary_ops(): input = r"""""" expected = [ Assignment(Variable('$a'), UnaryOp('-', 5), False), Assignment(Variable('$b'), UnaryOp('+', 6), False), Assignment(Variable('$c'), UnaryOp('!', Variable('$d')), False), Assignment(Variable('$e'), UnaryOp('~', Variable('$f')), False), ] eq_ast(input, expected) def test_assignment_ops(): input = r"""""" expected = [ AssignOp('+=', Variable('$a'), 5), AssignOp('-=', Variable('$b'), 6), AssignOp('.=', Variable('$c'), Variable('$d')), AssignOp('^=', Variable('$e'), Variable('$f')), ] eq_ast(input, expected) def test_object_properties(): input = r"""property; $object->foreach; $object->$variable; $object->$variable->schmariable; $object->$variable->$schmariable; ?>""" expected = [ ObjectProperty(Variable('$object'), 'property'), ObjectProperty(Variable('$object'), 'foreach'), ObjectProperty(Variable('$object'), Variable('$variable')), ObjectProperty(ObjectProperty(Variable('$object'), Variable('$variable')), 'schmariable'), ObjectProperty(ObjectProperty(Variable('$object'), Variable('$variable')), Variable('$schmariable')), ] eq_ast(input, expected) def test_string_unescape(): input = r"""""" # TODO: "\x97\x[0-9]"; expected = [ r"\r\n\t\'", "\r\n\t\\\"", ] eq_ast(input, expected) def test_string_offset_lookups(): input = r"""property"; "$too->many->properties"; "$adjacent->object$lookup"; "$two->$variables"; "stray -> [ ]"; "not[array]"; "non->object"; ?>""" expected = [ ArrayOffset(Variable('$array'), 'offset'), ArrayOffset(Variable('$array'), 42), ArrayOffset(Variable('$array'), Variable('$variable')), ArrayOffset(Variable('$curly'), 'offset'), BinaryOp('.', ArrayOffset(Variable('$too'), 'many'), '[offsets]'), BinaryOp('.', ArrayOffset(Variable('$next'), 'to'), Variable('$array')), ObjectProperty(Variable('$object'), 'property'), BinaryOp('.', ObjectProperty(Variable('$too'), 'many'), '->properties'), BinaryOp('.', ObjectProperty(Variable('$adjacent'), 'object'), Variable('$lookup')), BinaryOp('.', BinaryOp('.', Variable('$two'), '->'), Variable('$variables')), 'stray -> [ ]', 'not[array]', 'non->object', ] eq_ast(input, expected) def test_string_curly_dollar_expressions(): input = r"""items[4]->five}"; "{${$nasty}}"; "{${funcall()}}"; "{${$object->method()}}"; "{$object->$variable}"; "{$object->$variable[1]}"; "{${static_class::constant}}"; "{${static_class::$variable}}"; ?>""" expected = [ BinaryOp('.', BinaryOp('.', 'a', Variable('$dollar_curly')), 'b'), BinaryOp('.', BinaryOp('.', 'c', Variable('$curly_dollar')), 'd'), BinaryOp('.', BinaryOp('.', 'e', Variable('$dollar_curly_dollar')), 'f'), ArrayOffset(ArrayOffset(Variable('$array'), 0), 1), ArrayOffset(ArrayOffset(Variable('$array'), 'two'), 3), ObjectProperty(ArrayOffset(ObjectProperty(Variable('$object'), 'items'), 4), 'five'), Variable(Variable('$nasty')), Variable(FunctionCall('funcall', [])), Variable(MethodCall(Variable('$object'), 'method', [])), ObjectProperty(Variable('$object'), Variable('$variable')), ObjectProperty(Variable('$object'), ArrayOffset(Variable('$variable'), 1)), Variable(StaticProperty('static_class', 'constant')), Variable(StaticProperty('static_class', Variable('$variable'))), ] eq_ast(input, expected) def test_heredoc(): input = r"""variables. This is not the EOT; this is: EOT; ?>""" expected = [ Echo([BinaryOp('.', BinaryOp('.', BinaryOp('.', BinaryOp('.', 'This is a "', Variable('$heredoc')), '" with some '), ObjectProperty(Variable('$embedded'), 'variables')), '.\nThis is not the EOT; this is:')]), ] eq_ast(input, expected) if sys.version_info[0] < 3: eq_ast(input.decode('utf-8'), expected) def test_heredoc_no_var(): input = r"""""" expected = [ Echo(['This is a long\nheredoc without\nany variable.']) ] eq_ast(input, expected) if sys.version_info[0] < 3: eq_ast(input.decode('utf-8'), expected) def test_function_calls(): input = r"""""" expected = [ FunctionCall('f', []), FunctionCall('doit', [Parameter(Variable('$arg1'), False), Parameter(Variable('$arg2'), True), Parameter(BinaryOp('+', 3, 4), False)]), FunctionCall('name\\spaced', []), FunctionCall('\\name\\spaced', []), FunctionCall('namespace\\d', []), ] eq_ast(input, expected) def test_method_calls(): input = r"""meth($a, &$b, $c . $d); $chain->one($x)->two(&$y); ?>""" expected = [ MethodCall(Variable('$obj'), 'meth', [Parameter(Variable('$a'), False), Parameter(Variable('$b'), True), Parameter(BinaryOp('.', Variable('$c'), Variable('$d')), False)]), MethodCall(MethodCall(Variable('$chain'), 'one', [Parameter(Variable('$x'), False)]), 'two', [Parameter(Variable('$y'), True)]), ] eq_ast(input, expected) def test_if(): input = r""" $b) { return 1; } elseif ($a == $b) { return 0; } else { return 'firetruck'; } if ($if): echo 'a'; elseif ($elseif): echo 'b'; else: echo 'c'; endif; ?>""" expected = [ If(1, If(2, Echo([3]), [], Else(Echo([4]))), [], Else(Echo([5]))), If(BinaryOp('<', Variable('$a'), Variable('$b')), Block([Return(UnaryOp('-', 1))]), [ElseIf(BinaryOp('>', Variable('$a'), Variable('$b')), Block([Return(1)])), ElseIf(BinaryOp('==', Variable('$a'), Variable('$b')), Block([Return(0)]))], Else(Block([Return('firetruck')]))), If(Variable('$if'), Block([Echo(['a'])]), [ElseIf(Variable('$elseif'), Block([Echo(['b'])]))], Else(Block([Echo(['c'])]))), ] eq_ast(input, expected) def test_foreach(): input = r""" $eggs) { echo "$ham: $eggs"; } foreach (complex($expression) as &$ref) $ref++; foreach ($what as $de => &$dealy): yo(); yo(); endforeach; foreach ($foo as $bar[0]) {} ?>""" expected = [ Foreach(Variable('$foo'), None, ForeachVariable(Variable('$bar'), False), Block([Echo([Variable('$bar')])])), Foreach(Variable('$spam'), Variable('$ham'), ForeachVariable(Variable('$eggs'), False), Block([Echo([BinaryOp('.', BinaryOp('.', Variable('$ham'), ': '), Variable('$eggs'))])])), Foreach(FunctionCall('complex', [Parameter(Variable('$expression'), False)]), None, ForeachVariable(Variable('$ref'), True), PostIncDecOp('++', Variable('$ref'))), Foreach(Variable('$what'), Variable('$de'), ForeachVariable(Variable('$dealy'), True), Block([FunctionCall('yo', []), FunctionCall('yo', [])])), Foreach(Variable('$foo'), None, ForeachVariable(ArrayOffset(Variable('$bar'), 0), False), Block([])), ] eq_ast(input, expected) def test_foreach_with_lists(): input = r""" list($bar, $baz)) {} ?>""" expected = [ Foreach(Variable('$foo'), None, ForeachVariable([Variable('$bar'), Variable('$baz')], False), Block([])), Foreach(Variable('$foo'), Variable('$k'), ForeachVariable([Variable('$bar'), Variable('$baz')], False), Block([])), ] eq_ast(input, expected) def test_global_variables(): input = r"""prop}; ?>""" expected = [ Global([Variable('$foo'), Variable('$bar')]), Global([Variable(Variable('$yo'))]), Global([Variable(Variable('$dawg'))]), Global([Variable(ObjectProperty(Variable('$obj'), 'prop'))]), ] eq_ast(input, expected) def test_variable_variables(): input = r"""b; $$$triple; ?>""" expected = [ Assignment(Variable(Variable('$a')), Variable(Variable('$b')), False), Assignment(Variable(Variable('$a')), Variable(Variable('$b')), True), Assignment(Variable(Variable('$a')), Variable(Variable('$b')), False), Assignment(Variable(Variable('$a')), Variable(Variable('$b')), True), ObjectProperty(Variable(Variable('$a')), 'b'), Variable(Variable(Variable('$triple'))), ] eq_ast(input, expected) def test_classes(): input = r"""""" expected = [ Class('Clown', 'final', 'Unicycle', ['RedNose', 'FacePaint'], [], [ ClassConstants([ClassConstant('the', 'only'), ClassConstant('constant', 'is')]), ClassConstants([ClassConstant('change', 'chump')]), ClassVariables([], [ClassVariable('$iable', 999), ClassVariable('$nein', None)]), ClassVariables(['protected', 'static'], [ClassVariable('$x', None)]), Method('conjunction_junction', ['public'], [FormalParameter('$arg1', None, False, None), FormalParameter('$arg2', None, False, None)], [Return(BinaryOp('.', Variable('$arg1'), Variable('$arg2')))], False), ]), Class('Stub', None, None, [], [], []), ] eq_ast(input, expected) def test_new(): input = r"""""" expected = [ New('Foo', []), New('Foo', []), New('Bar', [Parameter(1, False), Parameter(2, False), Parameter(3, False)]), Assignment(Variable('$crusty'), New('OldSyntax', []), True), New('name\\Spaced', []), New('\\name\\Spaced', []), New('namespace\\D', []), ] eq_ast(input, expected) def test_exceptions(): input = r"""""" expected = [ Try([ Assignment(Variable('$a'), BinaryOp('+', Variable('$b'), Variable('$c')), False), Throw(New('Food', [Parameter(Variable('$a'), False)])), ], [ Catch('Food', Variable('$f'), [ Echo([BinaryOp('.', 'Received food: ', Variable('$f'))]) ]), Catch('\\Bar\\Food', Variable('$f'), [ Echo([BinaryOp('.', 'Received bar food: ', Variable('$f'))]) ]), Catch('namespace\\Food', Variable('$f'), [ Echo([BinaryOp('.', 'Received namespace food: ', Variable('$f'))]) ]), Catch('Exception', Variable('$e'), [ Echo(['Problem?']), ]), ], None) ] eq_ast(input, expected) def test_catch_finally(): input = r"""""" expected = [ Try([ 1 ], [ Catch('Exception', Variable('$e'), [ 2 ]), ], Finally([3])) ] eq_ast(input, expected) def test_just_finally(): input = r"""""" expected = [ Try([ ], [], Finally([1])) ] eq_ast(input, expected) def test_declare(): input = r"""""" expected = [ Declare([Directive('ticks', 1)], Block([ Echo(['hi']), ])), Declare([Directive('ticks', 2)], None), Declare([Directive('ticks', 3)], Block([ Echo(['bye']), ])), ] eq_ast(input, expected) def test_instanceof(): input = r"""""" expected = [ If(BinaryOp('instanceof', Variable('$foo'), Constant('Bar')), Block([Echo(['$foo is a bar'])]), [], None), BinaryOp('instanceof', Variable('$foo'), Variable('$bar')), BinaryOp('instanceof', Variable('$foo'), 'static'), ] eq_ast(input, expected) def test_static_members(): input = r"""""" expected = [ StaticProperty('Ztatic', 'constant'), StaticProperty('Ztatic', Variable('$variable')), StaticMethodCall('Ztatic', 'method', []), StaticMethodCall('Ztatic', Variable('$variable_method'), []), StaticProperty('static', 'late_binding'), StaticProperty('static', Variable('$late_binding')), StaticMethodCall('static', 'late_binding', []), ] eq_ast(input, expected) def test_casts(): input = r"""""" expected = [ Cast('array', Variable('$x')), Cast('bool', Variable('$x')), Cast('bool', Variable('$x')), Cast('double', Variable('$x')), Cast('double', Variable('$x')), Cast('double', Variable('$x')), Cast('int', Variable('$x')), Cast('int', Variable('$x')), Cast('string', Variable('$x')), Cast('unset', Variable('$x')), Cast('binary', Variable('$x')), ] eq_ast(input, expected) def test_namespaces(): input = r"""""" expected = [ Namespace('my\\name', []), Namespace('my\\name', [FunctionCall('foo', []), FunctionCall('bar', [])]), Namespace(None, [FunctionCall('foo', []), FunctionCall('bar', [])]), ] eq_ast(input, expected) def test_use_declarations(): input = r"""""" expected = [ UseDeclarations([UseDeclaration('me', None)]), UseDeclarations([UseDeclaration('\\me', None)]), UseDeclarations([UseDeclaration('\\me\\please', None)]), UseDeclarations([UseDeclaration('my\\name', 'foo')]), UseDeclarations([UseDeclaration('a', None), UseDeclaration('b', None)]), UseDeclarations([UseDeclaration('a', 'b'), UseDeclaration('\\c\\d\\e', 'f')]), ] eq_ast(input, expected) def test_constant_declarations(): input = r"""""" expected = [ ConstantDeclarations([ConstantDeclaration('foo', 42)]), ConstantDeclarations([ConstantDeclaration('bar', 'baz'), ConstantDeclaration('wat', Constant('\\DOO'))]), ConstantDeclarations([ConstantDeclaration('ant', Constant('namespace\\level'))]), ConstantDeclarations([ConstantDeclaration('dq1', '')]), ConstantDeclarations([ConstantDeclaration('dq2', 'nothing fancy')]), ] eq_ast(input, expected) def test_closures(): input = r"""""" expected = [ Assignment(Variable('$greet'), Closure([FormalParameter('$name', None, False, None)], [], [FunctionCall('printf', [Parameter('Hello %s\r\n', False), Parameter(Variable('$name'), False)])], False), False), FunctionCall(Variable('$greet'), [Parameter('World', False)]), Assignment(Variable('$cb'), Closure([FormalParameter('$a', None, False, None), FormalParameter('$b', None, True, None)], [LexicalVariable('$c', False), LexicalVariable('$d', True)], [], True), False), ] eq_ast(input, expected) def test_magic_constants(): input = r"""""" expected = [ Namespace('Shmamespace', []), Function('p', [FormalParameter('$x', None, False, None)], [ Echo([BinaryOp('.', BinaryOp('.', BinaryOp('.', MagicConstant('__FUNCTION__', 'Shmamespace\\p'), ': '), Variable('$x')), '\n')]) ], False), Class('Bar', None, None, [], [], [Method('__construct', [], [], [FunctionCall('p', [Parameter(MagicConstant('__LINE__', 10), False)]), FunctionCall('p', [Parameter(MagicConstant('__DIR__', '/my/dir'), False)]), FunctionCall('p', [Parameter(MagicConstant('__FILE__', '/my/dir/file.php'), False)]), FunctionCall('p', [Parameter(MagicConstant('__NAMESPACE__', 'Shmamespace'), False)]), FunctionCall('p', [Parameter(MagicConstant('__CLASS__', 'Shmamespace\\Bar'), False)]), FunctionCall('p', [Parameter(MagicConstant('__METHOD__', 'Shmamespace\\Bar::__construct'), False)])], False)]), New('Bar', []), ] eq_ast(input, expected, filename='/my/dir/file.php') def test_type_hinting(): input = r""""""; expected = [ Function('foo', [FormalParameter('$var1', None, False, 'Foo'), FormalParameter('$var2', 1, False, 'Bar'), FormalParameter('$var3', None, True, 'Quux'), FormalParameter('$var4', 1, True, 'Corge'), FormalParameter('$var5', Array([]), True, 'array')], [], False)] eq_ast(input, expected) def test_static_scalar_class_constants(): input = r"""""" expected = [ Class('A', None, None, [], [], [ClassVariables(['public'], [ClassVariable('$b', StaticProperty('self', 'C'))]), Method('d', [], [FormalParameter('$var1', StaticProperty('self', 'C'), False, None)], [], False) ])] eq_ast(input, expected) def test_backtick_shell_exec(): input = '' expected = [ BinaryOp('.', FunctionCall('shell_exec', [Parameter(Variable('$cmd'), False)]), FunctionCall('shell_exec', [Parameter('date', False)]) ), FunctionCall('shell_exec', [Parameter(BinaryOp('.', 'echo ', Variable('$line')), False)]) ] eq_ast(input, expected) def test_open_close_tags_ignore(): # The filtered lexer should correctly interpret ?>method()[0]; func()[1];' expected = [ ArrayOffset(MethodCall(Variable('$a'), 'method', []), 0), ArrayOffset(FunctionCall('func', []), 1) ] eq_ast(input, expected) def test_array_literal(): input = 'm1())->m2(); ($a->m1())->m2;''' expected = [ MethodCall(MethodCall(Variable('$a'), 'm1', []), 'm2', []), ObjectProperty(MethodCall(Variable('$a'), 'm1', []), 'm2'), ] eq_ast(input, expected) def test_binary_string(): input = '''m()[1][2]; $o->m(){1}{2}; ''' expected = [ ArrayOffset(ArrayOffset(MethodCall(Variable('$o'), 'm', []), 1), 2), StringOffset(StringOffset(MethodCall(Variable('$o'), 'm', []), 1), 2), ] eq_ast(input, expected) def test_yield(): input = '''f(); and `this` EOT; ?>""" expected = [ Echo(['disregard $all {$crazy} ${stuff}->f();\nand `this`']) ] eq_ast(input, expected) def test_exit_loc(): input = ''' syntax (does anyone use this?) states = ( ('php', 'exclusive'), ('quoted', 'exclusive'), ('quotedvar', 'exclusive'), ('varname', 'exclusive'), ('offset', 'exclusive'), ('property', 'exclusive'), ('heredoc', 'exclusive'), ('heredocvar', 'exclusive'), ('nowdoc', 'exclusive'), ('backticked', 'exclusive'), ('backtickedvar', 'exclusive'), ) # Reserved words reserved = ( 'ARRAY', 'AS', 'BREAK', 'CASE', 'CLASS', 'CONST', 'CONTINUE', 'DECLARE', 'DEFAULT', 'DIE', 'DO', 'ECHO', 'ELSE', 'ELSEIF', 'EMPTY', 'ENDDECLARE', 'ENDFOR', 'ENDFOREACH', 'ENDIF', 'ENDSWITCH', 'ENDWHILE', 'EVAL', 'EXIT', 'EXTENDS', 'FOR', 'FOREACH', 'FUNCTION', 'GLOBAL', 'IF', 'INCLUDE', 'INCLUDE_ONCE', 'INSTANCEOF', 'ISSET', 'LIST', 'NEW', 'PRINT', 'REQUIRE', 'REQUIRE_ONCE', 'RETURN', 'STATIC', 'SWITCH', 'UNSET', 'USE', 'VAR', 'WHILE', 'FINAL', 'INTERFACE', 'IMPLEMENTS', 'PUBLIC', 'PRIVATE', 'PROTECTED', 'ABSTRACT', 'CLONE', 'TRY', 'CATCH', 'THROW', 'NAMESPACE', 'FINALLY', 'TRAIT', 'YIELD', ) # Not used by parser unparsed = ( # Invisible characters 'WHITESPACE', # Open and close tags 'OPEN_TAG', 'OPEN_TAG_WITH_ECHO', 'CLOSE_TAG', # Comments 'COMMENT', 'DOC_COMMENT', ) tokens = reserved + unparsed + ( # Operators 'PLUS', 'MINUS', 'MUL', 'DIV', 'MOD', 'AND', 'OR', 'NOT', 'XOR', 'SL', 'SR', 'BOOLEAN_AND', 'BOOLEAN_OR', 'BOOLEAN_NOT', 'IS_SMALLER', 'IS_GREATER', 'IS_SMALLER_OR_EQUAL', 'IS_GREATER_OR_EQUAL', 'IS_EQUAL', 'IS_NOT_EQUAL', 'IS_IDENTICAL', 'IS_NOT_IDENTICAL', # Assignment operators 'EQUALS', 'MUL_EQUAL', 'DIV_EQUAL', 'MOD_EQUAL', 'PLUS_EQUAL', 'MINUS_EQUAL', 'SL_EQUAL', 'SR_EQUAL', 'AND_EQUAL', 'OR_EQUAL', 'XOR_EQUAL', 'CONCAT_EQUAL', # Increment/decrement 'INC', 'DEC', # Arrows 'OBJECT_OPERATOR', 'DOUBLE_ARROW', 'DOUBLE_COLON', # Delimiters 'LPAREN', 'RPAREN', 'LBRACKET', 'RBRACKET', 'LBRACE', 'RBRACE', 'DOLLAR', 'COMMA', 'CONCAT', 'QUESTION', 'COLON', 'SEMI', 'AT', 'NS_SEPARATOR', # Casts 'ARRAY_CAST', 'BINARY_CAST', 'BOOL_CAST', 'DOUBLE_CAST', 'INT_CAST', 'OBJECT_CAST', 'STRING_CAST', 'UNSET_CAST', # Escaping from HTML 'INLINE_HTML', # Identifiers and reserved words 'DIR', 'FILE', 'LINE', 'FUNC_C', 'CLASS_C', 'METHOD_C', 'NS_C', 'LOGICAL_AND', 'LOGICAL_OR', 'LOGICAL_XOR', 'HALT_COMPILER', 'STRING', 'VARIABLE', 'LNUMBER', 'DNUMBER', 'NUM_STRING', 'CONSTANT_ENCAPSED_STRING', 'ENCAPSED_AND_WHITESPACE', 'QUOTE', 'DOLLAR_OPEN_CURLY_BRACES', 'STRING_VARNAME', 'CURLY_OPEN', # Heredocs 'START_HEREDOC', 'END_HEREDOC', # Nowdocs 'START_NOWDOC', 'END_NOWDOC', # Backtick 'BACKTICK', ) # Newlines def t_php_WHITESPACE(t): r'[ \t\r\n]+' t.lexer.lineno += t.value.count("\n") return t # Operators t_php_PLUS = r'\+' t_php_MINUS = r'-' t_php_MUL = r'\*' t_php_DIV = r'/' t_php_MOD = r'%' t_php_AND = r'&' t_php_OR = r'\|' t_php_NOT = r'~' t_php_XOR = r'\^' t_php_SL = r'<<' t_php_SR = r'>>' t_php_BOOLEAN_AND = r'&&' t_php_BOOLEAN_OR = r'\|\|' t_php_BOOLEAN_NOT = r'!' t_php_IS_SMALLER = r'<' t_php_IS_GREATER = r'>' t_php_IS_SMALLER_OR_EQUAL = r'<=' t_php_IS_GREATER_OR_EQUAL = r'>=' t_php_IS_EQUAL = r'==' t_php_IS_NOT_EQUAL = r'(!=(?!=))|(<>)' t_php_IS_IDENTICAL = r'===' t_php_IS_NOT_IDENTICAL = r'!==' # Assignment operators t_php_EQUALS = r'=' t_php_MUL_EQUAL = r'\*=' t_php_DIV_EQUAL = r'/=' t_php_MOD_EQUAL = r'%=' t_php_PLUS_EQUAL = r'\+=' t_php_MINUS_EQUAL = r'-=' t_php_SL_EQUAL = r'<<=' t_php_SR_EQUAL = r'>>=' t_php_AND_EQUAL = r'&=' t_php_OR_EQUAL = r'\|=' t_php_XOR_EQUAL = r'\^=' t_php_CONCAT_EQUAL = r'\.=' # Increment/decrement t_php_INC = r'\+\+' t_php_DEC = r'--' # Arrows t_php_DOUBLE_ARROW = r'=>' t_php_DOUBLE_COLON = r'::' def t_php_OBJECT_OPERATOR(t): r'->' if re.match(r'[A-Za-z_]', peek(t.lexer)): t.lexer.push_state('property') return t # Delimeters t_php_LPAREN = r'\(' t_php_RPAREN = r'\)' t_php_DOLLAR = r'\$' t_php_COMMA = r',' t_php_CONCAT = r'\.(?!\d|=)' t_php_QUESTION = r'\?' t_php_COLON = r':' t_php_SEMI = r';' t_php_AT = r'@' t_php_NS_SEPARATOR = r'\\' def t_php_LBRACKET(t): r'\[' t.lexer.push_state('php') return t def t_php_RBRACKET(t): r'\]' t.lexer.pop_state() return t def t_php_LBRACE(t): r'\{' t.lexer.push_state('php') return t def t_php_RBRACE(t): r'\}' t.lexer.pop_state() return t # Casts t_php_ARRAY_CAST = r'\([ \t]*[Aa][Rr][Rr][Aa][Yy][ \t]*\)' t_php_BINARY_CAST = r'\([ \t]*[Bb][Ii][Nn][Aa][Rr][Yy][ \t]*\)' t_php_BOOL_CAST = r'\([ \t]*[Bb][Oo][Oo][Ll]([Ee][Aa][Nn])?[ \t]*\)' t_php_DOUBLE_CAST = r'\([ \t]*([Rr][Ee][Aa][Ll]|[Dd][Oo][Uu][Bb][Ll][Ee]|[Ff][Ll][Oo][Aa][Tt])[ \t]*\)' t_php_INT_CAST = r'\([ \t]*[Ii][Nn][Tt]([Ee][Gg][Ee][Rr])?[ \t]*\)' t_php_OBJECT_CAST = r'\([ \t]*[Oo][Bb][Jj][Ee][Cc][Tt][ \t]*\)' t_php_STRING_CAST = r'\([ \t]*[Ss][Tt][Rr][Ii][Nn][Gg][ \t]*\)' t_php_UNSET_CAST = r'\([ \t]*[Uu][Nn][Ss][Ee][Tt][ \t]*\)' # Comments def t_php_DOC_COMMENT(t): r'/\*\*(.|\n)*?\*/' t.lexer.lineno += t.value.count("\n") return t def t_php_COMMENT(t): r'/\*(.|\n)*?\*/ | //([^?%\n]|[?%](?!>))*\n? | \#([^?%\n]|[?%](?!>))*\n?' t.lexer.lineno += t.value.count("\n") return t # Escaping from HTML def t_OPEN_TAG(t): r'<[?%](([Pp][Hh][Pp][ \t\r\n]?)|=)?' if '=' in t.value: t.type = 'OPEN_TAG_WITH_ECHO' t.lexer.lineno += t.value.count("\n") t.lexer.begin('php') return t def t_php_CLOSE_TAG(t): r'[?%]>\r?\n?' t.lexer.lineno += t.value.count("\n") t.lexer.begin('INITIAL') return t def t_INLINE_HTML(t): r'([^<]|<(?![?%]))+' t.lexer.lineno += t.value.count("\n") return t # Identifiers and reserved words reserved_map = { '__DIR__': 'DIR', '__FILE__': 'FILE', '__LINE__': 'LINE', '__FUNCTION__': 'FUNC_C', '__CLASS__': 'CLASS_C', '__METHOD__': 'METHOD_C', '__NAMESPACE__': 'NS_C', 'AND': 'LOGICAL_AND', 'OR': 'LOGICAL_OR', 'XOR': 'LOGICAL_XOR', '__HALT_COMPILER': 'HALT_COMPILER', } for r in reserved: reserved_map[r] = r # Identifier def t_php_STRING(t): r'[A-Za-z_][\w_]*' t.type = reserved_map.get(t.value.upper(), 'STRING') return t # Variable def t_php_VARIABLE(t): r'\$[A-Za-z_][\w_]*' return t # Floating literal def t_php_DNUMBER(t): r'(\d*\.\d+|\d+\.\d*)([Ee][+-]?\d+)? | (\d+[Ee][+-]?\d+)' return t # Integer literal def t_php_LNUMBER(t): r'(0b[01]+)|(0x[0-9A-Fa-f]+)|\d+' return t # String literal def t_php_CONSTANT_ENCAPSED_STRING(t): r"'([^\\']|\\(.|\n))*'" t.lexer.lineno += t.value.count("\n") return t def t_php_QUOTE(t): r'"' t.lexer.push_state('quoted') return t def t_quoted_QUOTE(t): r'"' t.lexer.pop_state() return t def t_quoted_ENCAPSED_AND_WHITESPACE(t): r'( [^"\\${] | \\(.|\n) | \$(?![A-Za-z_{]) | \{(?!\$) )+' t.lexer.lineno += t.value.count("\n") return t def t_quoted_VARIABLE(t): r'\$[A-Za-z_][\w_]*' t.lexer.push_state('quotedvar') return t def t_quoted_CURLY_OPEN(t): r'\{(?=\$)' t.lexer.push_state('php') return t def t_quoted_DOLLAR_OPEN_CURLY_BRACES(t): r'\$\{' if re.match(r'[A-Za-z_]', peek(t.lexer)): t.lexer.push_state('varname') else: t.lexer.push_state('php') return t def t_quotedvar_QUOTE(t): r'"' t.lexer.pop_state() t.lexer.pop_state() return t def t_quotedvar_LBRACKET(t): r'\[' t.lexer.begin('offset') return t def t_quotedvar_OBJECT_OPERATOR(t): r'->(?=[A-Za-z])' t.lexer.begin('property') return t def t_quotedvar_ENCAPSED_AND_WHITESPACE(t): r'( [^"\\${] | \\(.|\n) | \$(?![A-Za-z_{]) | \{(?!\$) )+' t.lexer.lineno += t.value.count("\n") t.lexer.pop_state() return t t_quotedvar_VARIABLE = t_php_VARIABLE def t_quotedvar_CURLY_OPEN(t): r'\{(?=\$)' t.lexer.begin('php') return t def t_quotedvar_DOLLAR_OPEN_CURLY_BRACES(t): r'\$\{' if re.match(r'[A-Za-z_]', peek(t.lexer)): t.lexer.begin('varname') else: t.lexer.begin('php') return t def t_varname_STRING_VARNAME(t): r'[A-Za-z_][\w_]*' return t t_varname_RBRACE = t_php_RBRACE t_varname_LBRACKET = t_php_LBRACKET def t_offset_STRING(t): r'[A-Za-z_][\w_]*' return t def t_offset_NUM_STRING(t): r'\d+' return t t_offset_VARIABLE = t_php_VARIABLE t_offset_RBRACKET = t_php_RBRACKET def t_property_STRING(t): r'[A-Za-z_][\w_]*' t.lexer.pop_state() return t # Heredocs def t_php_START_HEREDOC(t): r'<<<[ \t]*(?P