From fbd08e099fcc3259bc799b8b3ae7c0c079f3af79 Mon Sep 17 00:00:00 2001 From: Tommy Parnell Date: Sun, 31 Dec 2017 21:27:26 -0500 Subject: [PATCH] parse integer literals, identifiers, expressions --- ast/ast.go | 80 +++++++++++++++++++++++++++++++++++- ast/ast_test.go | 28 +++++++++++++ parser/parser.go | 95 +++++++++++++++++++++++++++++++++++++++++-- parser/parser_test.go | 92 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 290 insertions(+), 5 deletions(-) create mode 100644 ast/ast_test.go diff --git a/ast/ast.go b/ast/ast.go index 799322b..c98bb3b 100644 --- a/ast/ast.go +++ b/ast/ast.go @@ -1,6 +1,9 @@ package ast -import "monkey/token" +import ( + "bytes" + "monkey/token" +) type LetStatement struct { Token token.Token // the token.LET token @@ -21,6 +24,7 @@ func (i *Identifier) TokenLiteral() string { return i.Token.Literal } type Node interface { TokenLiteral() string + String() string } type Statement interface { @@ -37,6 +41,11 @@ type Program struct { Statements []Statement } +type IntegerLiteral struct { + Token token.Token + Value int64 +} + func (p *Program) TokenLiteral() string { if len(p.Statements) > 0 { return p.Statements[0].TokenLiteral() @@ -44,3 +53,72 @@ func (p *Program) TokenLiteral() string { return "" } } + +func (il *IntegerLiteral) expressionNode() {} +func (il *IntegerLiteral) TokenLiteral() string { return il.Token.Literal } +func (il *IntegerLiteral) String() string { return il.Token.Literal } + +type ExpressionStatement struct { + Token token.Token // the first token of the expression + Expression Expression +} + +func (es *ExpressionStatement) statementNode() {} +func (es *ExpressionStatement) TokenLiteral() string { return es.Token.Literal } + +func (p *Program) String() string { + var out bytes.Buffer + + for _, s := range p.Statements { + out.WriteString(s.String()) + } + + return out.String() +} + +type ReturnStatement struct { + Token token.Token // the 'return' token + ReturnValue Expression +} + +func (rs *ReturnStatement) statementNode() {} +func (rs *ReturnStatement) TokenLiteral() string { return rs.Token.Literal } + +func (ls *LetStatement) String() string { + var out bytes.Buffer + + out.WriteString(ls.TokenLiteral() + " ") + out.WriteString(ls.Name.String()) + out.WriteString(" = ") + + if ls.Value != nil { + out.WriteString(ls.Value.String()) + } + + out.WriteString(";") + + return out.String() +} + +func (rs *ReturnStatement) String() string { + var out bytes.Buffer + + out.WriteString(rs.TokenLiteral() + " ") + + if rs.ReturnValue != nil { + out.WriteString(rs.ReturnValue.String()) + } + + out.WriteString(";") + + return out.String() +} + +func (es *ExpressionStatement) String() string { + if es.Expression != nil { + return es.Expression.String() + } + return "" +} + +func (i *Identifier) String() string { return i.Value } diff --git a/ast/ast_test.go b/ast/ast_test.go new file mode 100644 index 0000000..14a49dc --- /dev/null +++ b/ast/ast_test.go @@ -0,0 +1,28 @@ +package ast + +import ( + "monkey/token" + "testing" +) + +func TestString(t *testing.T) { + program := &Program{ + Statements: []Statement{ + &LetStatement{ + Token: token.Token{Type: token.LET, Literal: "let"}, + Name: &Identifier{ + Token: token.Token{Type: token.IDENT, Literal: "myVar"}, + Value: "myVar", + }, + Value: &Identifier{ + Token: token.Token{Type: token.IDENT, Literal: "anotherVar"}, + Value: "anotherVar", + }, + }, + }, + } + + if program.String() != "let myVar = anotherVar;" { + t.Errorf("program.String() wrong. got=%q", program.String()) + } +} diff --git a/parser/parser.go b/parser/parser.go index cb41c26..f2a5070 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -5,14 +5,42 @@ import ( "monkey/ast" "monkey/lexer" "monkey/token" + "strconv" +) + +const ( + _ int = iota + LOWEST + EQUALS // == + LESSGREATER // > or < + SUM // + + PRODUCT // * + PREFIX // -X or !X + CALL // myFunction(X) ) type Parser struct { - l *lexer.Lexer + l *lexer.Lexer + errors []string curToken token.Token peekToken token.Token - errors []string + + prefixParseFns map[token.TokenType]prefixParseFn + infixParseFns map[token.TokenType]infixParseFn +} + +type ( + prefixParseFn func() ast.Expression + infixParseFn func(ast.Expression) ast.Expression +) + +func (p *Parser) registerPrefix(tokenType token.TokenType, fn prefixParseFn) { + p.prefixParseFns[tokenType] = fn +} + +func (p *Parser) registerInfix(tokenType token.TokenType, fn infixParseFn) { + p.infixParseFns[tokenType] = fn } func New(l *lexer.Lexer) *Parser { @@ -36,7 +64,9 @@ func (p *Parser) nextToken() { func (p *Parser) ParseProgram() *ast.Program { program := &ast.Program{} program.Statements = []ast.Statement{} - + p.prefixParseFns = make(map[token.TokenType]prefixParseFn) + p.registerPrefix(token.IDENT, p.parseIdentifier) + p.registerPrefix(token.INT, p.parseIntegerLiteral) for p.curToken.Type != token.EOF { stmt := p.parseStatement() if stmt != nil { @@ -51,11 +81,27 @@ func (p *Parser) parseStatement() ast.Statement { switch p.curToken.Type { case token.LET: return p.parseLetStatement() + case token.RETURN: + return p.parseReturnStatement() default: - return nil + return p.parseExpressionStatement() } } +func (p *Parser) parseReturnStatement() *ast.ReturnStatement { + stmt := &ast.ReturnStatement{Token: p.curToken} + + p.nextToken() + + // TODO: We're skipping the expressions until we + // encounter a semicolon + for !p.curTokenIs(token.SEMICOLON) { + p.nextToken() + } + + return stmt +} + func (p *Parser) parseLetStatement() *ast.LetStatement { stmt := &ast.LetStatement{Token: p.curToken} @@ -78,6 +124,43 @@ func (p *Parser) parseLetStatement() *ast.LetStatement { return stmt } +func (p *Parser) parseExpressionStatement() *ast.ExpressionStatement { + stmt := &ast.ExpressionStatement{Token: p.curToken} + + stmt.Expression = p.parseExpression(LOWEST) + + if p.peekTokenIs(token.SEMICOLON) { + p.nextToken() + } + + return stmt +} + +func (p *Parser) parseExpression(precedence int) ast.Expression { + prefix := p.prefixParseFns[p.curToken.Type] + if prefix == nil { + return nil + } + leftExp := prefix() + + return leftExp +} + +func (p *Parser) parseIntegerLiteral() ast.Expression { + lit := &ast.IntegerLiteral{Token: p.curToken} + + value, err := strconv.ParseInt(p.curToken.Literal, 0, 64) + if err != nil { + msg := fmt.Sprintf("could not parse %q as integer", p.curToken.Literal) + p.errors = append(p.errors, msg) + return nil + } + + lit.Value = value + + return lit +} + func (p *Parser) curTokenIs(t token.TokenType) bool { return p.curToken.Type == t } @@ -104,3 +187,7 @@ func (p *Parser) peekError(t token.TokenType) { t, p.peekToken.Type) p.errors = append(p.errors, msg) } + +func (p *Parser) parseIdentifier() ast.Expression { + return &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal} +} diff --git a/parser/parser_test.go b/parser/parser_test.go index 9dbe14c..ded6c8e 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -79,3 +79,95 @@ func testLetStatement(t *testing.T, s ast.Statement, name string) bool { return true } + +func TestReturnStatements(t *testing.T) { + input := ` +return 5; +return 10; +return 993322; +` + l := lexer.New(input) + p := New(l) + + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 3 { + t.Fatalf("program.Statements does not contain 3 statements. got=%d", + len(program.Statements)) + } + + for _, stmt := range program.Statements { + returnStmt, ok := stmt.(*ast.ReturnStatement) + if !ok { + t.Errorf("stmt not *ast.returnStatement. got=%T", stmt) + continue + } + if returnStmt.TokenLiteral() != "return" { + t.Errorf("returnStmt.TokenLiteral not 'return', got %q", + returnStmt.TokenLiteral()) + } + } +} + +func TestIdentifierExpression(t *testing.T) { + input := "foobar;" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 1 { + t.Fatalf("program has not enough statements. got=%d", + len(program.Statements)) + } + stmt, ok := program.Statements[0].(*ast.ExpressionStatement) + if !ok { + t.Fatalf("program.Statements[0] is not ast.ExpressionStatement. got=%T", + program.Statements[0]) + } + + ident, ok := stmt.Expression.(*ast.Identifier) + if !ok { + t.Fatalf("exp not *ast.Identifier. got=%T", stmt.Expression) + } + if ident.Value != "foobar" { + t.Errorf("ident.Value not %s. got=%s", "foobar", ident.Value) + } + if ident.TokenLiteral() != "foobar" { + t.Errorf("ident.TokenLiteral not %s. got=%s", "foobar", + ident.TokenLiteral()) + } +} + +func TestIntegerLiteralExpression(t *testing.T) { + input := "5;" + + l := lexer.New(input) + p := New(l) + program := p.ParseProgram() + checkParserErrors(t, p) + + if len(program.Statements) != 1 { + t.Fatalf("program has not enough statements. got=%d", + len(program.Statements)) + } + stmt, ok := program.Statements[0].(*ast.ExpressionStatement) + if !ok { + t.Fatalf("program.Statements[0] is not ast.ExpressionStatement. got=%T", + program.Statements[0]) + } + + literal, ok := stmt.Expression.(*ast.IntegerLiteral) + if !ok { + t.Fatalf("exp not *ast.IntegerLiteral. got=%T", stmt.Expression) + } + if literal.Value != 5 { + t.Errorf("literal.Value not %d. got=%d", 5, literal.Value) + } + if literal.TokenLiteral() != "5" { + t.Errorf("literal.TokenLiteral not %s. got=%s", "5", + literal.TokenLiteral()) + } +}