From 45f6929c14614f6f18651b0a16f0eddb5f592d58 Mon Sep 17 00:00:00 2001 From: tijani Date: Mon, 30 May 2022 17:40:03 +0000 Subject: [PATCH] Started implementing Pratt Parsing, tests now pass that can generate on Infix and Integer expressions. git-svn-id: https://svn.tlawal.org/svn/monkey@23 f6afcba9-9ef1-4bdd-9b72-7484f5705bac --- ast/ast.go | 16 ++++++++++ parser/parser.go | 70 ++++++++++++++++++++++++++++++++++++++++++- parser/parser_test.go | 63 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 148 insertions(+), 1 deletion(-) diff --git a/ast/ast.go b/ast/ast.go index 881ffde..041b438 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -53,6 +53,11 @@ type ExpressionStatement struct { Expression Expression } +type IntegerLiteral struct { + Token token.Token + Value int64 +} + // Let Statements func (ls *LetStatement) statement_node() {} @@ -123,3 +128,14 @@ func (es *ExpressionStatement) String() string { } return "" } + +// IntegerLiteral +func (il *IntegerLiteral) expression_node() {} + +func (il *IntegerLiteral) TokenLiteral() string { + return il.Token.Literal +} + +func (il *IntegerLiteral) String() string { + return il.Token.Literal +} diff --git a/parser/parser.go b/parser/parser.go index a9f1c98..2749ef2 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -5,6 +5,25 @@ import ( "monkey/ast" "monkey/lexer" "monkey/token" + "strconv" +) + +// PRECEDENCE of operations +const ( + _ int = iota // iota means start from 0, hence _ starts from 0 + LOWEST + EQUALS // == + LESSGREATER // > OR < + SUM // + + PRODUCT // * + PREFIX // -x OR !x + CALL // simple_function(x) +) + +// Pratt Parsing +type ( + prefix_parse_function func() ast.Expression + infix_parse_function func(ast.Expression) ast.Expression ) type Parser struct { @@ -13,6 +32,9 @@ type Parser struct { peek_token token.Token errors []string + + prefix_parse_functions map[token.TokenType]prefix_parse_function + infix_parse_functions map[token.TokenType]infix_parse_function } func New(l_lexer *lexer.Lexer) *Parser { @@ -22,6 +44,10 @@ func New(l_lexer *lexer.Lexer) *Parser { l_parser.next_token() l_parser.next_token() + l_parser.prefix_parse_functions = make(map[token.TokenType]prefix_parse_function) + l_parser.register_prefix(token.IDENT, l_parser.parse_identifier) + l_parser.register_prefix(token.INT, l_parser.parse_integer_literal) + return l_parser } @@ -43,6 +69,18 @@ func (l_parser *Parser) ParseProgram() *ast.Program { return program } +func (l_parser *Parser) register_prefix(l_token_type token.TokenType, l_function prefix_parse_function) { + l_parser.prefix_parse_functions[l_token_type] = l_function +} + +func (l_parser *Parser) parse_identifier() ast.Expression { + return &ast.Identifier{Token: l_parser.current_token, Value: l_parser.current_token.Literal} +} + +func (l_parser *Parser) register_infix(l_token_type token.TokenType, l_function infix_parse_function) { + l_parser.infix_parse_functions[l_token_type] = l_function +} + func (l_parser *Parser) next_token() { l_parser.current_token = l_parser.peek_token l_parser.peek_token = l_parser.lexer.NextToken() @@ -55,7 +93,7 @@ func (l_parser *Parser) parse_statement() ast.Statement { case token.RETURN: return l_parser.parse_return_statement() default: - return nil + return l_parser.parse_expression_statement() } } @@ -88,6 +126,36 @@ func (l_parser *Parser) parse_return_statement() *ast.ReturnStatement { return statement } +func (l_parser *Parser) parse_expression_statement() *ast.ExpressionStatement { + statement := &ast.ExpressionStatement{Token: l_parser.current_token} + statement.Expression = l_parser.parse_expression(LOWEST) + if l_parser.peek_token_is(token.SEMICOLON) { + l_parser.next_token() + } + return statement +} + +func (l_parser *Parser) parse_integer_literal() ast.Expression { + literal := &ast.IntegerLiteral{Token: l_parser.current_token} + value, error := strconv.ParseInt(l_parser.current_token.Literal, 0, 64) + if error != nil { + message := fmt.Sprintf("could not parse %q as integer", l_parser.current_token.Literal) + l_parser.errors = append(l_parser.errors, message) + return nil + } + literal.Value = value + return literal +} + +func (l_parser *Parser) parse_expression(precedence int) ast.Expression { + prefix := l_parser.prefix_parse_functions[l_parser.current_token.Type] + if prefix == nil { + return nil + } + left_expression := prefix() + return left_expression +} + func (l_parser *Parser) expect_peek(l_token token.TokenType) bool { if l_parser.peek_token_is(l_token) { l_parser.next_token() diff --git a/parser/parser_test.go b/parser/parser_test.go index 0e81f0c..c91179a 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -112,3 +112,66 @@ func TestReturnStatement(l_test *testing.T) { } } } + +func TestIdentifierExpression(l_test *testing.T) { + input := "foobar;" + + l_lexer := lexer.New(input) + l_parser := New(l_lexer) + program := l_parser.ParseProgram() + check_parser_errors(l_test, l_parser) + + if len(program.Statements) != 1 { + l_test.Fatalf("program does not have enough staments, got=%d", len(program.Statements)) + } + + statement, ok := program.Statements[0].(*ast.ExpressionStatement) + if !ok { + l_test.Fatalf("program.Statements[0] is not ast.ExpressionStatement, got=%T", program.Statements[0]) + } + + identifier, ok := statement.Expression.(*ast.Identifier) + if !ok { + l_test.Fatalf("expression not *ast.Identifier, got=%T", statement.Expression) + } + + if identifier.Value != "foobar" { + l_test.Errorf("identifier.Value not %s, got=%s", "foobar", identifier.Value) + } + + if identifier.TokenLiteral() != "foobar" { + l_test.Errorf("identifier.TokenLiteral not %s, got=%s", "foobar", identifier.TokenLiteral()) + } +} + +func TestIntegerLiteralExpressions(l_test *testing.T) { + input := "5;" + + l_lexer := lexer.New(input) + l_parser := New(l_lexer) + program := l_parser.ParseProgram() + + check_parser_errors(l_test, l_parser) + + if len(program.Statements) != 1 { + l_test.Fatalf("program does not have enough statements, got=%d", len(program.Statements)) + } + + statement, ok := program.Statements[0].(*ast.ExpressionStatement) + if !ok { + l_test.Fatalf("program.Statements[0] is not ast.ExpressionStatement, got=%T", program.Statements[0]) + } + + literal, ok := statement.Expression.(*ast.IntegerLiteral) + if !ok { + l_test.Fatalf("expression not *ast.IntegerLiteral, got=%T", statement.Expression) + } + + if literal.Value != 5 { + l_test.Errorf("literal.Value not %d, got=%d", 5, literal.Value) + } + + if literal.TokenLiteral() != "5" { + l_test.Errorf("literal.TokenLiteral not %s, got=%s", "5", literal.TokenLiteral()) + } +}