From d8066a59ebdfaddc57f7c13184d5c6d9741a73ea Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 26 Jul 2024 09:53:29 +0200 Subject: [PATCH 01/84] stop consuming trivia as tokens --- runtime/parser/lexer/lexer.go | 51 ++++++++++++++++++++++++------ runtime/parser/lexer/lexer_test.go | 27 ++++++++++++++++ runtime/parser/lexer/state.go | 23 ++++++-------- runtime/parser/lexer/token.go | 4 +++ runtime/parser/lexer/tokentype.go | 1 + 5 files changed, 83 insertions(+), 23 deletions(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 7b69245ce2..f600426bf0 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -74,6 +74,8 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool + // currentTriviaRange stores the current leading or trailing trivia + currentTriviaRange *ast.Range } var _ TokenStream = &lexer{} @@ -273,17 +275,48 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co l.tokenCount = len(l.tokens) if consume { - l.startOffset = l.endOffset + l.consume(endPos) + } +} - l.startPos = endPos - r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) +func (l *lexer) emitTrivia(legacyTriviaType TokenType) { + fmt.Printf("Emitting trivia for %s\n", legacyTriviaType) - if r == '\n' { - l.startPos.line++ - l.startPos.column = 0 - } else { - l.startPos.column++ - } + endPos := l.endPos() + + currentRange := ast.NewRange( + l.memoryGauge, + l.startPosition(), + ast.NewPosition( + l.memoryGauge, + l.endOffset-1, + endPos.line, + endPos.column, + ), + ) + + if l.currentTriviaRange == nil { + l.currentTriviaRange = ¤tRange + } else { + // Extend the ending range when encountering more trivia + l.currentTriviaRange.EndPos = currentRange.EndPos + } + + l.consume(endPos) +} + +// endPos pre-computed end-position by calling l.endPos() +func (l *lexer) consume(endPos position) { + l.startOffset = l.endOffset + + l.startPos = endPos + r, _ := utf8.DecodeRune(l.input[l.endOffset-1:]) + + if r == '\n' { + l.startPos.line++ + l.startPos.column = 0 + } else { + l.startPos.column++ } } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 51f8f53f34..519c1efcdd 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -873,6 +873,33 @@ func TestLexBasic(t *testing.T) { }, ) }) + + t.Run("trivia", func(t *testing.T) { + testLex(t, + "// test is in next line \n/* test is here */ test", + []token{ + { + Token: Token{ + Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, + EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, + }, + }, + Source: "test", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + EndPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + }, + }, + }, + }, + ) + }) } func TestLexString(t *testing.T) { diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 4b252d6f09..822a04b898 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -121,10 +121,11 @@ func rootState(l *lexer) stateFn { case '/': r = l.next() switch r { + // TODO(preserve-comments): Deprecate trivia token types case '/': return lineCommentState case '*': - l.emitType(TokenBlockCommentStart) + l.emitTrivia(TokenBlockCommentStart) return blockCommentState(0) default: l.backupOne() @@ -267,14 +268,8 @@ func spaceState(startIsNewline bool) stateFn { common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - l.emit( - TokenSpace, - Space{ - ContainsNewline: containsNewline, - }, - l.startPosition(), - true, - ) + l.emitTrivia(TokenSpace) + return rootState } } @@ -307,7 +302,7 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitType(TokenLineComment) + l.emitTrivia(TokenLineComment) return rootState } @@ -326,9 +321,9 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('*') { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitType(TokenBlockCommentContent) + l.emitTrivia(TokenBlockCommentContent) l.endOffset = starOffset - l.emitType(TokenBlockCommentStart) + l.emitTrivia(TokenBlockCommentStart) return blockCommentState(nesting + 1) } @@ -337,9 +332,9 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('/') { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitType(TokenBlockCommentContent) + l.emitTrivia(TokenBlockCommentContent) l.endOffset = slashOffset - l.emitType(TokenBlockCommentEnd) + l.emitTrivia(TokenBlockCommentEnd) return blockCommentState(nesting - 1) } } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index cfda861bdf..874f573c09 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -26,6 +26,10 @@ type Token struct { SpaceOrError any ast.Range Type TokenType + // leading trivia up to and including the first contiguous sequence of newlines characters. + LeadingTrivia string + // trailing trivia up to, but not including, the next newline character. + TrailingTrivia string } func (t Token) Is(ty TokenType) bool { diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index 0a15c19b6f..0c190b6edb 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -69,6 +69,7 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual + // TODO(preserve-comments): Deprecate trivia token types TokenBlockCommentStart TokenBlockCommentEnd TokenBlockCommentContent From 76f7107fb09ac02dd1dd786be5eb43dd23c9220f Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 26 Jul 2024 11:43:32 +0200 Subject: [PATCH 02/84] compute leading and trailing trivia --- runtime/parser/lexer/lexer.go | 38 ++++++++++++++++++++++++++++++ runtime/parser/lexer/lexer_test.go | 10 +++++--- 2 files changed, 45 insertions(+), 3 deletions(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index f600426bf0..bae7cb7e5d 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -101,6 +101,9 @@ func (l *lexer) Next() Token { pos, pos, ), + LeadingTrivia: string(l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset]), + // EOF has no trailing trivia + TrailingTrivia: "", } } @@ -190,6 +193,8 @@ func (l *lexer) run(state stateFn) { for state != nil { state = state(l) } + + l.updatePreviousTrailingTrivia() } // next decodes the next rune (UTF8 character) from the input string. @@ -256,6 +261,8 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos := l.endPos() + l.updatePreviousTrailingTrivia() + token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -269,8 +276,14 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), + LeadingTrivia: string(l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset]), + // Trailing trivia can't be determined, as it wasn't consumed yet at this point. + TrailingTrivia: "", } + // Mark as fully consumed + l.currentTriviaRange = nil + l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) @@ -279,6 +292,31 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co } } +func (l *lexer) updatePreviousTrailingTrivia() { + if l.tokenCount > 0 { + leadingTriviaRange := l.previousTokenTrailingTriviaRange() + l.tokens[l.tokenCount-1].TrailingTrivia = string(l.input[leadingTriviaRange.StartPos.Offset:leadingTriviaRange.EndPos.Offset]) + // Consume part of the current trivia + l.currentTriviaRange.StartPos = leadingTriviaRange.EndPos + } +} + +func (l *lexer) previousTokenTrailingTriviaRange() ast.Range { + trivia := l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset] + + endPos := l.currentTriviaRange.StartPos + for _, r := range trivia { + if r == '\n' { + break + } else { + endPos.Column++ + endPos.Offset++ + } + } + + return ast.NewRange(l.memoryGauge, l.currentTriviaRange.StartPos, endPos) +} + func (l *lexer) emitTrivia(legacyTriviaType TokenType) { fmt.Printf("Emitting trivia for %s\n", legacyTriviaType) diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 519c1efcdd..db30a42bc8 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -876,7 +876,7 @@ func TestLexBasic(t *testing.T) { t.Run("trivia", func(t *testing.T) { testLex(t, - "// test is in next line \n/* test is here */ test", + "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", []token{ { Token: Token{ @@ -885,6 +885,8 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, }, + LeadingTrivia: "// test is in next line \n/* test is here */ ", + TrailingTrivia: " // test is in the same line", }, Source: "test", }, @@ -892,9 +894,11 @@ func TestLexBasic(t *testing.T) { Token: Token{ Type: TokenEOF, Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 23, Offset: 48}, - EndPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, }, + LeadingTrivia: "\n// test is in previous line", + TrailingTrivia: "", }, }, }, From 95438a8be2f368551e72cd225f974cb0b28f6d71 Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 26 Jul 2024 12:56:55 +0200 Subject: [PATCH 03/84] partially fix trivia sourcing --- runtime/ast/position.go | 6 ++++++ runtime/parser/lexer/lexer.go | 23 +++++++++++++++++------ runtime/parser/lexer/token.go | 4 +--- 3 files changed, 24 insertions(+), 9 deletions(-) diff --git a/runtime/ast/position.go b/runtime/ast/position.go index 3a7dc42f89..1183985a9f 100644 --- a/runtime/ast/position.go +++ b/runtime/ast/position.go @@ -143,6 +143,12 @@ func (e Range) EndPosition(common.MemoryGauge) Position { return e.EndPos } +func (e Range) Source(input []byte) []byte { + startOffset := e.StartPos.Offset + endOffset := e.EndPos.Offset + 1 + return input[startOffset:endOffset] +} + // NewRangeFromPositioned func NewRangeFromPositioned(memoryGauge common.MemoryGauge, hasPosition HasPosition) Range { diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index bae7cb7e5d..5b39e40507 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -94,6 +94,11 @@ func (l *lexer) Next() Token { endPos.column, ) + var leadingTrivia string + if l.currentTriviaRange != nil { + leadingTrivia = string(l.currentTriviaRange.Source(l.input)) + } + return Token{ Type: TokenEOF, Range: ast.NewRange( @@ -101,7 +106,7 @@ func (l *lexer) Next() Token { pos, pos, ), - LeadingTrivia: string(l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset]), + LeadingTrivia: leadingTrivia, // EOF has no trailing trivia TrailingTrivia: "", } @@ -263,6 +268,11 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co l.updatePreviousTrailingTrivia() + var leadingTrivia string + if l.currentTriviaRange != nil { + leadingTrivia = string(l.currentTriviaRange.Source(l.input)) + } + token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -276,7 +286,7 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), - LeadingTrivia: string(l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset]), + LeadingTrivia: leadingTrivia, // Trailing trivia can't be determined, as it wasn't consumed yet at this point. TrailingTrivia: "", } @@ -293,22 +303,23 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co } func (l *lexer) updatePreviousTrailingTrivia() { - if l.tokenCount > 0 { + if l.tokenCount > 0 && l.currentTriviaRange != nil { leadingTriviaRange := l.previousTokenTrailingTriviaRange() - l.tokens[l.tokenCount-1].TrailingTrivia = string(l.input[leadingTriviaRange.StartPos.Offset:leadingTriviaRange.EndPos.Offset]) + l.tokens[l.tokenCount-1].TrailingTrivia = string(leadingTriviaRange.Source(l.input)) // Consume part of the current trivia l.currentTriviaRange.StartPos = leadingTriviaRange.EndPos } } func (l *lexer) previousTokenTrailingTriviaRange() ast.Range { - trivia := l.input[l.currentTriviaRange.StartPos.Offset:l.currentTriviaRange.EndPos.Offset] - + trivia := l.currentTriviaRange.Source(l.input) endPos := l.currentTriviaRange.StartPos + for _, r := range trivia { if r == '\n' { break } else { + // TODO(preserve-comments): Handle other trivia characters endPos.Column++ endPos.Offset++ } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 874f573c09..831a6b7510 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -37,7 +37,5 @@ func (t Token) Is(ty TokenType) bool { } func (t Token) Source(input []byte) []byte { - startOffset := t.StartPos.Offset - endOffset := t.EndPos.Offset + 1 - return input[startOffset:endOffset] + return t.Range.Source(input) } From a6619eb0a59d5a7f631650c5c28fa196c2ebe784 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 28 Jul 2024 10:58:54 +0200 Subject: [PATCH 04/84] add draft test case --- runtime/parser/lexer/lexer_test.go | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index db30a42bc8..e47ff60b81 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -906,6 +906,28 @@ func TestLexBasic(t *testing.T) { }) } +func TestLexTrivia(t *testing.T) { + + t.Parallel() + + // TODO(preserve-comments): Trailing trivia contains one character more than it should + t.Run("trivia", func(t *testing.T) { + testLex(t, ` +// This transaction calculates sum +transaction ( + // First operand 1 + a: Int, // First operand 2 + // Second operand 1 + b: Int // Second operand 2 +) { + // Logs the sum 1 + log(a, b) // Logs the sum 2 +} + +`, []token{}) // TODO(preserve-comments): Define expected output + }) +} + func TestLexString(t *testing.T) { t.Parallel() From 5cffdbf84d61faf5ce4b27877503bd291bedc046 Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 29 Jul 2024 11:28:05 +0200 Subject: [PATCH 05/84] test doc comments in AST --- runtime/parser/lexer/lexer.go | 32 +++++++++++++++++++++++++++++++- runtime/parser/parser_test.go | 34 ++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 1 deletion(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 5b39e40507..3d77d20ce8 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -328,8 +328,14 @@ func (l *lexer) previousTokenTrailingTriviaRange() ast.Range { return ast.NewRange(l.memoryGauge, l.currentTriviaRange.StartPos, endPos) } +const useLegacyTrivia = true + func (l *lexer) emitTrivia(legacyTriviaType TokenType) { - fmt.Printf("Emitting trivia for %s\n", legacyTriviaType) + // TODO(preserve-comments): Remove after refactoring is complete + if useLegacyTrivia { + l.emitTriviaLegacy(legacyTriviaType) + return + } endPos := l.endPos() @@ -354,6 +360,30 @@ func (l *lexer) emitTrivia(legacyTriviaType TokenType) { l.consume(endPos) } +func (l *lexer) emitTriviaLegacy(legacyType TokenType) { + if legacyType == TokenSpace { + trivia := l.input[l.startOffset:l.endOffset] + + var containsNewline bool + for _, r := range trivia { + if r == '\n' { + containsNewline = true + } + } + + l.emit( + TokenSpace, + Space{ + ContainsNewline: containsNewline, + }, + l.startPosition(), + true, + ) + } else { + l.emitType(legacyType) + } +} + // endPos pre-computed end-position by calling l.endPos() func (l *lexer) consume(endPos position) { l.startOffset = l.endOffset diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index ff816dd4c5..06532da432 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1013,3 +1013,37 @@ func TestParseWhitespaceAtEnd(t *testing.T) { assert.Empty(t, errs) } + +func TestParseTrivia(t *testing.T) { + + t.Parallel() + + t.Run("doc comments", func(t *testing.T) { + res, errs := ParseProgram( + nil, + []byte(` +/// Inline doc +fun first() {} + +/** +Multi-line doc +*/ +fun second() {} +`), + Config{}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, " Inline doc", first.DocString) + + second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, "\nMulti-line doc\n", second.DocString) + }) + + t.Run("non-doc comments", func(t *testing.T) {}) +} From 44c666b70b0275364b6d95893fb4b78fbaf52306 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 30 Jul 2024 13:58:47 +0200 Subject: [PATCH 06/84] refactor trivia parsing --- runtime/parser/lexer/lexer.go | 101 +++++++++++++---------------- runtime/parser/lexer/lexer_test.go | 49 ++++++++++++-- runtime/parser/lexer/state.go | 47 +++++++------- runtime/parser/lexer/token.go | 8 +-- runtime/parser/lexer/tokentype.go | 2 +- runtime/parser/lexer/trivia.go | 17 +++++ 6 files changed, 133 insertions(+), 91 deletions(-) create mode 100644 runtime/parser/lexer/trivia.go diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 3d77d20ce8..7faa5ef91e 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -74,8 +74,8 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool - // currentTriviaRange stores the current leading or trailing trivia - currentTriviaRange *ast.Range + // currentTrivia stores the current leading or trailing Trivia + currentTrivia []Trivia } var _ TokenStream = &lexer{} @@ -94,11 +94,6 @@ func (l *lexer) Next() Token { endPos.column, ) - var leadingTrivia string - if l.currentTriviaRange != nil { - leadingTrivia = string(l.currentTriviaRange.Source(l.input)) - } - return Token{ Type: TokenEOF, Range: ast.NewRange( @@ -106,9 +101,9 @@ func (l *lexer) Next() Token { pos, pos, ), - LeadingTrivia: leadingTrivia, - // EOF has no trailing trivia - TrailingTrivia: "", + LeadingTrivia: l.currentTrivia, + // EOF has no trailing Trivia + TrailingTrivia: []Trivia{}, } } @@ -268,11 +263,6 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co l.updatePreviousTrailingTrivia() - var leadingTrivia string - if l.currentTriviaRange != nil { - leadingTrivia = string(l.currentTriviaRange.Source(l.input)) - } - token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -286,13 +276,13 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), - LeadingTrivia: leadingTrivia, - // Trailing trivia can't be determined, as it wasn't consumed yet at this point. - TrailingTrivia: "", + LeadingTrivia: l.currentTrivia, + // Trailing Trivia can't be determined, as it wasn't consumed yet at this point. + TrailingTrivia: []Trivia{}, } // Mark as fully consumed - l.currentTriviaRange = nil + l.currentTrivia = []Trivia{} l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) @@ -303,40 +293,31 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co } func (l *lexer) updatePreviousTrailingTrivia() { - if l.tokenCount > 0 && l.currentTriviaRange != nil { - leadingTriviaRange := l.previousTokenTrailingTriviaRange() - l.tokens[l.tokenCount-1].TrailingTrivia = string(leadingTriviaRange.Source(l.input)) - // Consume part of the current trivia - l.currentTriviaRange.StartPos = leadingTriviaRange.EndPos + if l.tokenCount == 0 { + return } -} -func (l *lexer) previousTokenTrailingTriviaRange() ast.Range { - trivia := l.currentTriviaRange.Source(l.input) - endPos := l.currentTriviaRange.StartPos - - for _, r := range trivia { - if r == '\n' { - break + // Split the current trivia into trailing trivia of the previous token + // and leading trivia of the next token. + var trailingTrivia []Trivia + var leadingTrivia []Trivia + trailingTriviaEnded := false + for _, trivia := range l.currentTrivia { + if trivia.Type == TriviaTypeNewLine { + trailingTriviaEnded = true + } + if trailingTriviaEnded { + leadingTrivia = append(leadingTrivia, trivia) } else { - // TODO(preserve-comments): Handle other trivia characters - endPos.Column++ - endPos.Offset++ + trailingTrivia = append(trailingTrivia, trivia) } } - return ast.NewRange(l.memoryGauge, l.currentTriviaRange.StartPos, endPos) + l.tokens[l.tokenCount-1].TrailingTrivia = trailingTrivia + l.currentTrivia = leadingTrivia } -const useLegacyTrivia = true - -func (l *lexer) emitTrivia(legacyTriviaType TokenType) { - // TODO(preserve-comments): Remove after refactoring is complete - if useLegacyTrivia { - l.emitTriviaLegacy(legacyTriviaType) - return - } - +func (l *lexer) emitTrivia(triviaType TriviaType) { endPos := l.endPos() currentRange := ast.NewRange( @@ -350,18 +331,27 @@ func (l *lexer) emitTrivia(legacyTriviaType TokenType) { ), ) - if l.currentTriviaRange == nil { - l.currentTriviaRange = ¤tRange - } else { - // Extend the ending range when encountering more trivia - l.currentTriviaRange.EndPos = currentRange.EndPos + if l.currentTrivia == nil { + l.currentTrivia = []Trivia{} } + l.currentTrivia = append(l.currentTrivia, Trivia{ + Type: triviaType, + Text: currentRange.Source(l.input), + }) + l.consume(endPos) } -func (l *lexer) emitTriviaLegacy(legacyType TokenType) { - if legacyType == TokenSpace { +const useLegacyTrivia = false + +// TODO(preserve-comments): Remove after refactoring is complete +func (l *lexer) emitLegacyTrivia(legacyTriviaType TokenType) { + if !useLegacyTrivia { + return + } + + if legacyTriviaType == TokenSpace { trivia := l.input[l.startOffset:l.endOffset] var containsNewline bool @@ -380,7 +370,7 @@ func (l *lexer) emitTriviaLegacy(legacyType TokenType) { true, ) } else { - l.emitType(legacyType) + l.emitType(legacyTriviaType) } } @@ -449,16 +439,13 @@ func (l *lexer) emitError(err error) { l.emit(TokenError, err, rangeStart, false) } -func (l *lexer) scanSpace() (containsNewline bool) { +func (l *lexer) scanSpace() { // lookahead is already lexed. // parse more, if any l.acceptWhile(func(r rune) bool { switch r { case ' ', '\t', '\r': return true - case '\n': - containsNewline = true - return true default: return false } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index e47ff60b81..96251cbdc5 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -874,7 +874,7 @@ func TestLexBasic(t *testing.T) { ) }) - t.Run("trivia", func(t *testing.T) { + t.Run("Trivia", func(t *testing.T) { testLex(t, "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", []token{ @@ -885,8 +885,34 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, }, - LeadingTrivia: "// test is in next line \n/* test is here */ ", - TrailingTrivia: " // test is in the same line", + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeInlineComment, + Text: []byte("// test is in next line "), + }, + { + Type: TriviaTypeNewLine, + Text: []byte("\n"), + }, + { + Type: TriviaTypeMultiLineComment, + Text: []byte("/* test is here */"), + }, + { + Type: TriviaTypeSpace, + Text: []byte(" "), + }, + }, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Text: []byte(" "), + }, + { + Type: TriviaTypeInlineComment, + Text: []byte("// test is in the same line"), + }, + }, }, Source: "test", }, @@ -897,8 +923,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, }, - LeadingTrivia: "\n// test is in previous line", - TrailingTrivia: "", + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Text: []byte("\n"), + }, + { + Type: TriviaTypeInlineComment, + Text: []byte("// test is in previous line"), + }, + }, + TrailingTrivia: []Trivia{}, }, }, }, @@ -910,8 +945,8 @@ func TestLexTrivia(t *testing.T) { t.Parallel() - // TODO(preserve-comments): Trailing trivia contains one character more than it should - t.Run("trivia", func(t *testing.T) { + // TODO(preserve-comments): Trailing Trivia contains one character more than it should + t.Run("Trivia", func(t *testing.T) { testLex(t, ` // This transaction calculates sum transaction ( diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 822a04b898..fb1a013a48 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -111,9 +111,11 @@ func rootState(l *lexer) stateFn { case '_': return identifierState case ' ', '\t', '\r': - return spaceState(false) + return spaceState case '\n': - return spaceState(true) + l.emitLegacyTrivia(TokenSpace) + l.emitTrivia(TriviaTypeNewLine) + return rootState case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': return numberState case '"': @@ -121,12 +123,12 @@ func rootState(l *lexer) stateFn { case '/': r = l.next() switch r { - // TODO(preserve-comments): Deprecate trivia token types + // TODO(preserve-comments): Deprecate Trivia token types case '/': return lineCommentState case '*': - l.emitTrivia(TokenBlockCommentStart) - return blockCommentState(0) + l.emitLegacyTrivia(TokenBlockCommentStart) + return blockCommentState(l, 0) default: l.backupOne() l.emitType(TokenSlash) @@ -261,17 +263,16 @@ type Space struct { ContainsNewline bool } -func spaceState(startIsNewline bool) stateFn { - return func(l *lexer) stateFn { - containsNewline := l.scanSpace() - containsNewline = containsNewline || startIsNewline +func spaceState(l *lexer) stateFn { + l.scanSpace() - common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) + // TODO(preserve-comments): Do we need to track memory for other token types as well? + common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - l.emitTrivia(TokenSpace) + l.emitLegacyTrivia(TokenSpace) + l.emitTrivia(TriviaTypeSpace) - return rootState - } + return rootState } func identifierState(l *lexer) stateFn { @@ -302,12 +303,14 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitTrivia(TokenLineComment) + l.emitLegacyTrivia(TokenLineComment) + l.emitTrivia(TriviaTypeInlineComment) return rootState } -func blockCommentState(nesting int) stateFn { +func blockCommentState(l *lexer, nesting int) stateFn { if nesting < 0 { + l.emitTrivia(TriviaTypeMultiLineComment) return rootState } @@ -321,10 +324,10 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('*') { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitTrivia(TokenBlockCommentContent) + l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = starOffset - l.emitTrivia(TokenBlockCommentStart) - return blockCommentState(nesting + 1) + l.emitLegacyTrivia(TokenBlockCommentStart) + return blockCommentState(l, nesting+1) } case '*': @@ -332,13 +335,13 @@ func blockCommentState(nesting int) stateFn { if l.acceptOne('/') { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitTrivia(TokenBlockCommentContent) + l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = slashOffset - l.emitTrivia(TokenBlockCommentEnd) - return blockCommentState(nesting - 1) + l.emitLegacyTrivia(TokenBlockCommentEnd) + return blockCommentState(l, nesting-1) } } - return blockCommentState(nesting) + return blockCommentState(l, nesting) } } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 831a6b7510..e7f68471f2 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -26,10 +26,10 @@ type Token struct { SpaceOrError any ast.Range Type TokenType - // leading trivia up to and including the first contiguous sequence of newlines characters. - LeadingTrivia string - // trailing trivia up to, but not including, the next newline character. - TrailingTrivia string + // leading Trivia up to and including the first contiguous sequence of newlines characters. + LeadingTrivia []Trivia + // trailing Trivia up to, but not including, the next newline character. + TrailingTrivia []Trivia } func (t Token) Is(ty TokenType) bool { diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index 0c190b6edb..dd5ced009c 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -69,7 +69,7 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual - // TODO(preserve-comments): Deprecate trivia token types + // TODO(preserve-comments): Deprecate Trivia token types TokenBlockCommentStart TokenBlockCommentEnd TokenBlockCommentContent diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go new file mode 100644 index 0000000000..cda47a93bc --- /dev/null +++ b/runtime/parser/lexer/trivia.go @@ -0,0 +1,17 @@ +package lexer + +type Trivia struct { + Type TriviaType + // The source text (includes opening/closing comment characters in case of comment trivia type) + Text []byte +} + +type TriviaType uint8 + +const ( + TriviaTypeUnknown TriviaType = iota + TriviaTypeInlineComment + TriviaTypeMultiLineComment + TriviaTypeNewLine + TriviaTypeSpace +) From 0539fa320bc657eaf9553bbb17289126d5181969 Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 31 Jul 2024 12:53:59 +0200 Subject: [PATCH 07/84] start tracking comments in ast, add sample tests --- runtime/ast/comments.go | 71 +++++++++++++++++++++++++++++ runtime/ast/function_declaration.go | 57 ++++++++++++++++++++--- runtime/ast/identifier.go | 1 - runtime/parser/function.go | 8 +++- runtime/parser/lexer/lexer.go | 4 +- runtime/parser/lexer/trivia.go | 6 ++- runtime/parser/parser.go | 31 +++++++++++++ runtime/parser/parser_test.go | 36 ++++++++++++--- 8 files changed, 196 insertions(+), 18 deletions(-) create mode 100644 runtime/ast/comments.go diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go new file mode 100644 index 0000000000..2504ee5bad --- /dev/null +++ b/runtime/ast/comments.go @@ -0,0 +1,71 @@ +package ast + +import ( + "bytes" + "github.com/onflow/cadence/runtime/common" +) + +type Comments struct { + Leading []Comment + Trailing []Comment +} + +type Comment struct { + source []byte +} + +func NewComment(memoryGauge common.MemoryGauge, source []byte) Comment { + // TODO(preserve-comments): Track memory usage + return Comment{ + source: source, + } +} + +var blockCommentDocStringPrefix = []byte("/**") +var blockCommentStringPrefix = []byte("/*") +var lineCommentDocStringPrefix = []byte("///") +var lineCommentStringPrefix = []byte("//") +var blockCommentStringSuffix = []byte("*/") + +func (c Comment) Multiline() bool { + return bytes.HasPrefix(c.source, blockCommentStringPrefix) +} + +func (c Comment) Doc() bool { + if c.Multiline() { + return bytes.HasPrefix(c.source, blockCommentDocStringPrefix) + } else { + return bytes.HasPrefix(c.source, lineCommentDocStringPrefix) + } +} + +// Text without opening/closing comment characters /*, /**, */, // +func (c Comment) Text() []byte { + withoutPrefixes := cutOptionalPrefixes(c.source, [][]byte{ + blockCommentDocStringPrefix, // check before blockCommentStringPrefix + blockCommentStringPrefix, + lineCommentDocStringPrefix, // check before lineCommentStringPrefix + lineCommentStringPrefix, + }) + return cutOptionalSuffixes(withoutPrefixes, [][]byte{ + blockCommentStringSuffix, + }) +} + +func cutOptionalPrefixes(input []byte, prefixes [][]byte) (output []byte) { + output = input + for _, prefix := range prefixes { + cut, _ := bytes.CutPrefix(output, prefix) + output = cut + } + return +} + +func cutOptionalSuffixes(input []byte, suffixes [][]byte) (output []byte) { + output = input + for _, suffix := range suffixes { + cut, _ := bytes.CutSuffix(output, suffix) + output = cut + } + return +} diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index f86f6be670..6bd75761c5 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -20,6 +20,7 @@ package ast import ( "encoding/json" + "strings" "github.com/turbolent/prettier" @@ -69,17 +70,52 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - DocString string - Identifier Identifier - StartPos Position `json:"-"` - Access Access - Flags FunctionDeclarationFlags + // TODO(preserve-comments): Replace with DeclarationDocString method + DocString string + Identifier Identifier + StartPos Position `json:"-"` + Access Access + Flags FunctionDeclarationFlags + Comments } var _ Element = &FunctionDeclaration{} var _ Declaration = &FunctionDeclaration{} var _ Statement = &FunctionDeclaration{} +func NewFunctionDeclarationWithComments( + gauge common.MemoryGauge, + access Access, + purity FunctionPurity, + isStatic bool, + isNative bool, + identifier Identifier, + typeParameterList *TypeParameterList, + parameterList *ParameterList, + returnTypeAnnotation *TypeAnnotation, + functionBlock *FunctionBlock, + startPos Position, + docString string, + comments Comments, +) *FunctionDeclaration { + decl := NewFunctionDeclaration( + gauge, + access, + purity, + isStatic, + isNative, + identifier, + typeParameterList, + parameterList, + returnTypeAnnotation, + functionBlock, + startPos, + docString, + ) + decl.Comments = comments + return decl +} + func NewFunctionDeclaration( gauge common.MemoryGauge, access Access, @@ -176,7 +212,16 @@ func (d *FunctionDeclaration) DeclarationMembers() *Members { } func (d *FunctionDeclaration) DeclarationDocString() string { - return d.DocString + var s strings.Builder + for _, comment := range d.Comments.Leading { + if comment.Doc() { + if s.Len() > 0 { + s.WriteRune('\n') + } + s.Write(comment.Text()) + } + } + return s.String() } func (d *FunctionDeclaration) Doc() prettier.Doc { diff --git a/runtime/ast/identifier.go b/runtime/ast/identifier.go index 515ea7699b..dff0b10739 100644 --- a/runtime/ast/identifier.go +++ b/runtime/ast/identifier.go @@ -20,7 +20,6 @@ package ast import ( "encoding/json" - "github.com/onflow/cadence/runtime/common" ) diff --git a/runtime/parser/function.go b/runtime/parser/function.go index a6028be802..cf3f1a40b6 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -328,9 +328,13 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { + var leadingTrivia []lexer.Trivia + var trailingTrivia []lexer.Trivia startPos := ast.EarliestPosition(p.current.StartPos, accessPos, purityPos, staticPos, nativePos) + leadingTrivia = append(leadingTrivia, p.current.LeadingTrivia...) + // Skip the `fun` keyword p.nextSemanticToken() @@ -353,6 +357,7 @@ func parseFunctionDeclaration( } } + // TODO(preserve-comments): Handle trailing comments parameterList, returnTypeAnnotation, functionBlock, err := parseFunctionParameterListAndRest(p, functionBlockIsOptional) @@ -360,7 +365,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, purity, @@ -373,6 +378,7 @@ func parseFunctionDeclaration( functionBlock, startPos, docString, + p.newCommentsFromTrivia(leadingTrivia, trailingTrivia), ), nil } diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 7faa5ef91e..9734b9c36e 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -336,8 +336,8 @@ func (l *lexer) emitTrivia(triviaType TriviaType) { } l.currentTrivia = append(l.currentTrivia, Trivia{ - Type: triviaType, - Text: currentRange.Source(l.input), + Type: triviaType, + Range: currentRange, }) l.consume(endPos) diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index cda47a93bc..4eb0dfa0e9 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -1,9 +1,11 @@ package lexer +import "github.com/onflow/cadence/runtime/ast" + type Trivia struct { Type TriviaType - // The source text (includes opening/closing comment characters in case of comment trivia type) - Text []byte + // Position within the source code (includes opening/closing comment characters in case of comment trivia type) + ast.Range } type TriviaType uint8 diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 497d1f54e4..6e5b64910a 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -286,6 +286,11 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } +func (p *parser) triviaSource(trivia lexer.Trivia) []byte { + input := p.tokens.Input() + return trivia.Source(input) +} + func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } @@ -553,6 +558,32 @@ func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { ) } +func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { + comments := ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + } + + for _, t := range leadingTrivia { + if isTriviaComment(t) { + comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + for _, t := range trailingTrivia { + if isTriviaComment(t) { + comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + return comments +} + +func isTriviaComment(t lexer.Trivia) bool { + // Other trivia types (space, newlines) are not needed in further stages of the compiler + return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment +} + func (p *parser) startAmbiguity() { if p.ambiguityLevel == 0 { p.localReplayedTokensCount = 0 diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index 06532da432..08e69de643 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1022,13 +1022,19 @@ func TestParseTrivia(t *testing.T) { res, errs := ParseProgram( nil, []byte(` -/// Inline doc -fun first() {} +/// Inline doc 1 of first +/// Inline doc 2 of first +fun first() {} // Trailing inline comment of first /** -Multi-line doc +Multi-line doc 1 of second +*/ +/** +Multi-line doc 2 of second +*/ +fun second() {} /** +Trailing multi-line comment of second */ -fun second() {} `), Config{}, ) @@ -1038,11 +1044,29 @@ fun second() {} first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) assert.True(t, ok) - assert.Equal(t, " Inline doc", first.DocString) + assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// Inline doc 1 of first")), + ast.NewComment(nil, []byte("/// Inline doc 2 of first")), + }, + Trailing: []ast.Comment{ + ast.NewComment(nil, []byte("// Trailing inline comment of first")), + }, + }, first.Comments) second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) assert.True(t, ok) - assert.Equal(t, "\nMulti-line doc\n", second.DocString) + assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), + ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), + }, + Trailing: []ast.Comment{ + ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), + }, + }, second.Comments) }) t.Run("non-doc comments", func(t *testing.T) {}) From 1fd5a6f63819f00de0e669fb9f73cf073264bd3b Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 31 Jul 2024 12:57:40 +0200 Subject: [PATCH 08/84] update comments --- runtime/ast/comments.go | 4 ++-- runtime/ast/function_declaration.go | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index 2504ee5bad..a8200104b6 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -42,9 +42,9 @@ func (c Comment) Doc() bool { // Text without opening/closing comment characters /*, /**, */, // func (c Comment) Text() []byte { withoutPrefixes := cutOptionalPrefixes(c.source, [][]byte{ - blockCommentDocStringPrefix, // check before blockCommentStringPrefix + blockCommentDocStringPrefix, // must be before blockCommentStringPrefix blockCommentStringPrefix, - lineCommentDocStringPrefix, // check before lineCommentStringPrefix + lineCommentDocStringPrefix, // must be before lineCommentStringPrefix lineCommentStringPrefix, }) return cutOptionalSuffixes(withoutPrefixes, [][]byte{ diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index 6bd75761c5..ce2ea941de 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -70,7 +70,7 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - // TODO(preserve-comments): Replace with DeclarationDocString method + // TODO(preserve-comments): Replace with DeclarationDocString method, since doc string is computed from Comments struct DocString string Identifier Identifier StartPos Position `json:"-"` @@ -83,6 +83,7 @@ var _ Element = &FunctionDeclaration{} var _ Declaration = &FunctionDeclaration{} var _ Statement = &FunctionDeclaration{} +// TODO(preserve-comments): Temporary, add `comments` param to NewFunctionDeclaration in the future func NewFunctionDeclarationWithComments( gauge common.MemoryGauge, access Access, From 96e7347c506f7cd1becb874f17ccd4eb2b6c0251 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 1 Aug 2024 15:05:57 +0200 Subject: [PATCH 09/84] parse block trailing comments --- runtime/ast/block.go | 8 ++++++++ runtime/parser/parser_test.go | 15 +++++++++++---- runtime/parser/statement.go | 3 ++- 3 files changed, 21 insertions(+), 5 deletions(-) diff --git a/runtime/ast/block.go b/runtime/ast/block.go index 1f1b70a596..69518f8077 100644 --- a/runtime/ast/block.go +++ b/runtime/ast/block.go @@ -29,10 +29,18 @@ import ( type Block struct { Statements []Statement Range + Comments } var _ Element = &Block{} +// TODO(preserve-comments): Migrate and remove +func NewBlockWithComments(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { + block := NewBlock(memoryGauge, statements, astRange) + block.Comments = comments + return block +} + func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index 08e69de643..a81ffaa1af 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1018,7 +1018,7 @@ func TestParseTrivia(t *testing.T) { t.Parallel() - t.Run("doc comments", func(t *testing.T) { + t.Run("function declaration", func(t *testing.T) { res, errs := ParseProgram( nil, []byte(` @@ -1050,10 +1050,14 @@ Trailing multi-line comment of second ast.NewComment(nil, []byte("/// Inline doc 1 of first")), ast.NewComment(nil, []byte("/// Inline doc 2 of first")), }, + Trailing: []ast.Comment{}, + }, first.Comments) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{}, Trailing: []ast.Comment{ ast.NewComment(nil, []byte("// Trailing inline comment of first")), }, - }, first.Comments) + }, first.FunctionBlock.Block.Comments) second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) assert.True(t, ok) @@ -1063,11 +1067,14 @@ Trailing multi-line comment of second ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), }, + Trailing: []ast.Comment{}, + }, second.Comments) + assert.Equal(t, ast.Comments{ + Leading: []ast.Comment{}, Trailing: []ast.Comment{ ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), }, - }, second.Comments) + }, second.FunctionBlock.Block.Comments) }) - t.Run("non-doc comments", func(t *testing.T) {}) } diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index cd5c9b93e4..35deb28838 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -556,7 +556,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { return ast.NewFunctionBlock( p.memoryGauge, - ast.NewBlock( + ast.NewBlockWithComments( p.memoryGauge, statements, ast.NewRange( @@ -564,6 +564,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + p.newCommentsFromTrivia(startToken.LeadingTrivia, endToken.TrailingTrivia), ), preConditions, postConditions, From ff4f9d8edad2df1528dc7dd569223dac41af1e7a Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 1 Aug 2024 15:48:01 +0200 Subject: [PATCH 10/84] update old parser, remove DocField field on FunctionDeclaration --- runtime/ast/comments.go | 4 +- runtime/ast/function_declaration.go | 31 +++++---- runtime/ast/function_declaration_test.go | 12 ++-- runtime/old_parser/declaration_test.go | 75 ++++++++++++++++----- runtime/old_parser/function.go | 7 +- runtime/old_parser/parser.go | 31 +++++++++ runtime/parser/declaration_test.go | 75 ++++++++++++++++----- runtime/parser/function.go | 11 +-- runtime/sema/check_composite_declaration.go | 2 +- runtime/sema/check_function.go | 2 +- runtime/sema/positioninfo.go | 2 +- 11 files changed, 180 insertions(+), 72 deletions(-) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index a8200104b6..4081e074df 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -6,8 +6,8 @@ import ( ) type Comments struct { - Leading []Comment - Trailing []Comment + Leading []Comment `json:"-"` + Trailing []Comment `json:"-"` } type Comment struct { diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index ce2ea941de..2614edba08 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -70,12 +70,10 @@ type FunctionDeclaration struct { ParameterList *ParameterList ReturnTypeAnnotation *TypeAnnotation FunctionBlock *FunctionBlock - // TODO(preserve-comments): Replace with DeclarationDocString method, since doc string is computed from Comments struct - DocString string - Identifier Identifier - StartPos Position `json:"-"` - Access Access - Flags FunctionDeclarationFlags + Identifier Identifier + StartPos Position `json:"-"` + Access Access + Flags FunctionDeclarationFlags Comments } @@ -151,7 +149,6 @@ func NewFunctionDeclaration( ReturnTypeAnnotation: returnTypeAnnotation, FunctionBlock: functionBlock, StartPos: startPos, - DocString: docString, } } @@ -246,16 +243,18 @@ func (d *FunctionDeclaration) MarshalJSON() ([]byte, error) { *Alias Type string Range - IsStatic bool - IsNative bool - Flags FunctionDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + Flags FunctionDeclarationFlags `json:",omitempty"` + DocString string }{ - Type: "FunctionDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Alias: (*Alias)(d), - Flags: 0, + Type: "FunctionDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Alias: (*Alias)(d), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/function_declaration_test.go b/runtime/ast/function_declaration_test.go index c164a67da4..521df5bf8b 100644 --- a/runtime/ast/function_declaration_test.go +++ b/runtime/ast/function_declaration_test.go @@ -113,8 +113,10 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, } actual, err := json.Marshal(decl) @@ -584,8 +586,10 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, }, - DocString: "test", - StartPos: Position{Offset: 34, Line: 35, Column: 36}, + Comments: Comments{ + Leading: []Comment{NewComment(nil, []byte("///test"))}, + }, + StartPos: Position{Offset: 34, Line: 35, Column: 36}, }, } diff --git a/runtime/old_parser/declaration_test.go b/runtime/old_parser/declaration_test.go index 1ecbc1e9bb..a009603a0d 100644 --- a/runtime/old_parser/declaration_test.go +++ b/runtime/old_parser/declaration_test.go @@ -816,8 +816,10 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -854,8 +856,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -892,8 +899,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -6870,8 +6881,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -6885,8 +6900,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -6910,8 +6929,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -6938,6 +6961,10 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 245, Line: 11, Column: 43}, EndPos: ast.Position{Offset: 246, Line: 11, Column: 44}, }, + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, }, }, StartPos: ast.Position{Offset: 216, Line: 11, Column: 14}, @@ -6988,8 +7015,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7006,8 +7037,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7024,8 +7059,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " destroyWithBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// destroyWithBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index 7d0054faaf..9e02144775 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -295,8 +295,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -329,7 +329,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclaration( + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, ast.FunctionPurityUnspecified, @@ -342,6 +342,7 @@ func parseFunctionDeclaration( functionBlock, startPos, docString, + p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/old_parser/parser.go b/runtime/old_parser/parser.go index 05b0d5bf84..299109886a 100644 --- a/runtime/old_parser/parser.go +++ b/runtime/old_parser/parser.go @@ -274,6 +274,11 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } +func (p *parser) triviaSource(trivia lexer.Trivia) []byte { + input := p.tokens.Input() + return trivia.Source(input) +} + func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } @@ -527,6 +532,32 @@ func (p *parser) endAmbiguity() { } } +func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { + comments := ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + } + + for _, t := range leadingTrivia { + if isTriviaComment(t) { + comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + for _, t := range trailingTrivia { + if isTriviaComment(t) { + comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) + } + } + + return comments +} + +func isTriviaComment(t lexer.Trivia) bool { + // Other trivia types (space, newlines) are not needed in further stages of the compiler + return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment +} + func ParseExpression( memoryGauge common.MemoryGauge, input []byte, diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 9f3544050f..a594b5db58 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -881,8 +881,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Test", - StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// Test")), + }, + }, + StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, result, @@ -919,8 +923,13 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " First line\n Second line", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// First line")), + ast.NewComment(nil, []byte("/// Second line")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -957,8 +966,12 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - DocString: " Cool dogs.\n\n Cool cats!! ", - StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), + }, + }, + StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, result, @@ -1368,7 +1381,10 @@ func TestParseFunctionDeclaration(t *testing.T) { PreConditions: (*ast.Conditions)(nil), PostConditions: (*ast.Conditions)(nil), }, - DocString: "", + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ @@ -3511,7 +3527,10 @@ func TestParseCompositeDeclaration(t *testing.T) { }, }, }, - DocString: "", + Comments: ast.Comments{ + Leading: []ast.Comment{}, + Trailing: []ast.Comment{}, + }, Identifier: ast.Identifier{ Identifier: "getFoo", Pos: ast.Position{ @@ -8889,8 +8908,12 @@ func TestParseMemberDocStrings(t *testing.T) { Members: ast.NewUnmeteredMembers( []ast.Declaration{ &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " noReturnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// noReturnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -8904,8 +8927,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 74, Line: 5, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -8929,8 +8956,12 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 143, Line: 8, Column: 14}, }, &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " returnAndBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// returnAndBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -9004,8 +9035,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " unknown", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// unknown")), + }, + }, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -9022,8 +9057,12 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - DocString: " initNoBlock", + Access: ast.AccessNotSpecified, + Comments: ast.Comments{ + Leading: []ast.Comment{ + ast.NewComment(nil, []byte("/// initNoBlock")), + }, + }, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, diff --git a/runtime/parser/function.go b/runtime/parser/function.go index cf3f1a40b6..e6e5cc0aae 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -328,12 +328,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - var leadingTrivia []lexer.Trivia - var trailingTrivia []lexer.Trivia - - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, purityPos, staticPos, nativePos) - - leadingTrivia = append(leadingTrivia, p.current.LeadingTrivia...) + startToken := p.current + startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() @@ -357,7 +353,6 @@ func parseFunctionDeclaration( } } - // TODO(preserve-comments): Handle trailing comments parameterList, returnTypeAnnotation, functionBlock, err := parseFunctionParameterListAndRest(p, functionBlockIsOptional) @@ -378,7 +373,7 @@ func parseFunctionDeclaration( functionBlock, startPos, docString, - p.newCommentsFromTrivia(leadingTrivia, trailingTrivia), + p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/sema/check_composite_declaration.go b/runtime/sema/check_composite_declaration.go index e6a42aa358..d2e4d45944 100644 --- a/runtime/sema/check_composite_declaration.go +++ b/runtime/sema/check_composite_declaration.go @@ -1845,7 +1845,7 @@ func (checker *Checker) defaultMembersAndOrigins( TypeAnnotation: fieldTypeAnnotation, VariableKind: ast.VariableKindConstant, ArgumentLabels: argumentLabels, - DocString: function.DocString, + DocString: function.DeclarationDocString(), HasImplementation: hasImplementation, HasConditions: hasConditions, }) diff --git a/runtime/sema/check_function.go b/runtime/sema/check_function.go index 4a716a204a..3ef555f3d2 100644 --- a/runtime/sema/check_function.go +++ b/runtime/sema/check_function.go @@ -147,7 +147,7 @@ func (checker *Checker) declareFunctionDeclaration( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: functionType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: common.DeclarationKindFunction, pos: declaration.Identifier.Pos, diff --git a/runtime/sema/positioninfo.go b/runtime/sema/positioninfo.go index 8c4b19dc5e..9b0a8a5687 100644 --- a/runtime/sema/positioninfo.go +++ b/runtime/sema/positioninfo.go @@ -111,7 +111,7 @@ func (i *PositionInfo) recordFunctionDeclarationOrigin( DeclarationKind: common.DeclarationKindFunction, StartPos: &startPosition, EndPos: &endPosition, - DocString: function.DocString, + DocString: function.DeclarationDocString(), } i.Occurrences.Put( From e4a107357e2d6c82238ba9c0a7c6220764def4ce Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 2 Aug 2024 09:41:52 +0200 Subject: [PATCH 11/84] fix existing lexer tests --- runtime/parser/lexer/lexer_test.go | 904 +++++++++++++---------------- runtime/parser/lexer/state.go | 1 + runtime/parser/lexer/token.go | 1 + 3 files changed, 401 insertions(+), 505 deletions(-) diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 96251cbdc5..d46a87156e 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -88,19 +88,6 @@ func TestLexBasic(t *testing.T) { testLex(t, " 01\t 10", []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -108,21 +95,26 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - Source: "01", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + }, + }, }, }, - Source: "\t ", + Source: "01", }, { Token: Token{ @@ -215,21 +207,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, }, }, - Source: " ", + Source: "2", }, { Token: Token{ @@ -238,21 +226,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "+", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "+", }, { Token: Token{ @@ -271,21 +255,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - }, - Source: ")", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, }, }, - Source: " ", + Source: ")", }, { Token: Token{ @@ -294,21 +274,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - }, - Source: "*", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, }, }, - Source: " ", + Source: "*", }, { Token: Token{ @@ -354,21 +330,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, }, }, - Source: " ", + Source: "2", }, { Token: Token{ @@ -377,21 +349,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "-", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "-", }, { Token: Token{ @@ -410,21 +378,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - }, - Source: ")", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, }, }, - Source: " ", + Source: ")", }, { Token: Token{ @@ -433,21 +397,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - }, - Source: "/", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, }, }, - Source: " ", + Source: "/", }, { Token: Token{ @@ -483,21 +443,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - Source: "1", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, - Source: " \n ", + Source: "1", }, { Token: Token{ @@ -506,21 +462,24 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 2, Offset: 5}, EndPos: ast.Position{Line: 2, Column: 2, Offset: 5}, }, - }, - Source: "2", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, - EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, + EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, + }, + }, }, }, - Source: "\n", + Source: "2", }, { Token: Token{ @@ -529,6 +488,15 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 0, Offset: 7}, EndPos: ast.Position{Line: 3, Column: 0, Offset: 7}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + }, + }, + }, }, }, }, @@ -546,21 +514,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - Source: "1", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, - Source: " ", + Source: "1", }, { Token: Token{ @@ -569,21 +533,17 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "??", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, }, }, - Source: " ", + Source: "??", }, { Token: Token{ @@ -874,7 +834,7 @@ func TestLexBasic(t *testing.T) { ) }) - t.Run("Trivia", func(t *testing.T) { + t.Run("trivia", func(t *testing.T) { testLex(t, "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", []token{ @@ -888,29 +848,95 @@ func TestLexBasic(t *testing.T) { LeadingTrivia: []Trivia{ { Type: TriviaTypeInlineComment, - Text: []byte("// test is in next line "), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + }, }, { Type: TriviaTypeNewLine, - Text: []byte("\n"), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + }, }, { Type: TriviaTypeMultiLineComment, - Text: []byte("/* test is here */"), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 25, + Line: 2, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 42, + Line: 2, + Column: 17, + }, + }, }, { Type: TriviaTypeSpace, - Text: []byte(" "), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 43, + Line: 2, + Column: 18, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 2, + Column: 18, + }, + }, }, }, TrailingTrivia: []Trivia{ { Type: TriviaTypeSpace, - Text: []byte(" "), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 48, + Line: 2, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 48, + Line: 2, + Column: 23, + }, + }, }, { Type: TriviaTypeInlineComment, - Text: []byte("// test is in the same line"), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 49, + Line: 2, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 75, + Line: 2, + Column: 50, + }, + }, }, }, }, @@ -926,11 +952,33 @@ func TestLexBasic(t *testing.T) { LeadingTrivia: []Trivia{ { Type: TriviaTypeNewLine, - Text: []byte("\n"), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 76, + Line: 2, + Column: 51, + }, + EndPos: ast.Position{ + Offset: 76, + Line: 2, + Column: 51, + }, + }, }, { Type: TriviaTypeInlineComment, - Text: []byte("// test is in previous line"), + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 77, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 103, + Line: 3, + Column: 26, + }, + }, }, }, TrailingTrivia: []Trivia{}, @@ -1116,19 +1164,6 @@ func TestLexString(t *testing.T) { }, Source: "\"", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1136,6 +1171,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + }, }, }, }, @@ -1156,19 +1200,6 @@ func TestLexString(t *testing.T) { }, Source: "\"te", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1176,6 +1207,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 4}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 4}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + }, }, }, }, @@ -1250,19 +1290,6 @@ func TestLexString(t *testing.T) { }, Source: "\"\\", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -1270,6 +1297,15 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 3}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + }, }, }, }, @@ -1287,53 +1323,14 @@ func TestLexBlockComment(t *testing.T) { []token{ { Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Source: ` // *X `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 10, Offset: 10}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 17, Offset: 17}, - }, - }, - Source: ` \\* `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, + Type: TokenError, + SpaceOrError: errors.New(`end of the file in a comment`), Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - EndPos: ast.Position{Line: 1, Column: 19, Offset: 19}, + StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, + EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, }, }, - Source: "*/", + Source: "\000", }, { Token: Token{ @@ -1352,89 +1349,6 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/* test foo /* bar */ asd */ `, []token{ - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: ` test foo `, - }, - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 14, Offset: 14}, - EndPos: ast.Position{Line: 1, Column: 18, Offset: 18}, - }, - }, - Source: ` bar `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 19, Offset: 19}, - EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenBlockCommentContent, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 21, Offset: 21}, - EndPos: ast.Position{Line: 1, Column: 25, Offset: 25}, - }, - }, - Source: ` asd `, - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 26, Offset: 26}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - }, - Source: "*/", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, - EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenEOF, @@ -1442,6 +1356,22 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, EndPos: ast.Position{Line: 1, Column: 30, Offset: 30}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeMultiLineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, + EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, + }, + }, + }, }, }, }, @@ -2067,19 +1997,6 @@ func TestLexIntegerLiterals(t *testing.T) { }, Source: "0", }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "\n", - }, { Token: Token{ Type: TokenEOF, @@ -2087,6 +2004,15 @@ func TestLexIntegerLiterals(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + }, }, }, }, @@ -2321,19 +2247,6 @@ func TestLexLineComment(t *testing.T) { testLex(t, ` foo // bar `, []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenIdentifier, @@ -2341,31 +2254,33 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "foo", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, }, - }, - Source: " ", - }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, }, }, - Source: "// bar ", + Source: "foo", }, { Token: Token{ @@ -2386,19 +2301,6 @@ func TestLexLineComment(t *testing.T) { t, " foo // bar \n baz", []token{ - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - Source: " ", - }, { Token: Token{ Type: TokenIdentifier, @@ -2406,44 +2308,33 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - }, - Source: "foo", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - Source: " ", - }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, - { - Token: Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: true, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + { + Type: TriviaTypeInlineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, }, }, - Source: "\n ", + Source: "foo", }, { Token: Token{ @@ -2452,6 +2343,22 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 1, Offset: 14}, EndPos: ast.Position{Line: 2, Column: 3, Offset: 16}, }, + LeadingTrivia: []Trivia{ + { + Type: TriviaTypeNewLine, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + }, + }, + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + }, + }, + }, }, Source: "baz", }, @@ -2484,19 +2391,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, tokenStream.Next(), @@ -2511,19 +2413,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, }, }, tokenStream.Next(), @@ -2550,6 +2447,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2561,6 +2459,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2578,19 +2477,14 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, }, }, tokenStream.Next(), @@ -2617,6 +2511,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2628,6 +2523,7 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2649,19 +2545,14 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - }, - tokenStream.Next(), - ) - - assert.Equal(t, - Token{ - Type: TokenSpace, - SpaceOrError: Space{ - ContainsNewline: false, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + TrailingTrivia: []Trivia{ + { + Type: TriviaTypeSpace, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, }, }, tokenStream.Next(), @@ -2691,6 +2582,7 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2715,6 +2607,7 @@ func TestEOFsAfterEmptyInput(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, + TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -2726,8 +2619,9 @@ func TestLimit(t *testing.T) { t.Parallel() var b strings.Builder + // TODO(preserve-comments): Token limit is affected, because comments are not treated as separate tokens for i := 0; i < 300000; i++ { - b.WriteString("x ") + b.WriteString("x x ") } code := b.String() diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index fb1a013a48..315f7afa29 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -318,6 +318,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { r := l.next() switch r { case EOF: + l.emitError(fmt.Errorf("end of the file in a comment")) return nil case '/': beforeSlashOffset := l.prevEndOffset diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index e7f68471f2..9982a3d616 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -23,6 +23,7 @@ import ( ) type Token struct { + // TODO(preserve-comments): Rename to Error error SpaceOrError any ast.Range Type TokenType From 4158ce86ede111c8cb0efddc03ad9493937af568 Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 13:03:28 +0200 Subject: [PATCH 12/84] introduce `isFollowedByTrivia` --- runtime/parser/lexer/token.go | 4 ++-- runtime/parser/lexer/trivia.go | 11 +++++++++++ runtime/parser/parser.go | 18 ++++++++++++++++++ runtime/parser/type.go | 14 +------------- 4 files changed, 32 insertions(+), 15 deletions(-) diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 9982a3d616..6b573619a4 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -28,9 +28,9 @@ type Token struct { ast.Range Type TokenType // leading Trivia up to and including the first contiguous sequence of newlines characters. - LeadingTrivia []Trivia + LeadingTrivia TriviaCollection // trailing Trivia up to, but not including, the next newline character. - TrailingTrivia []Trivia + TrailingTrivia TriviaCollection } func (t Token) Is(ty TokenType) bool { diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index 4eb0dfa0e9..4fd702098b 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -17,3 +17,14 @@ const ( TriviaTypeNewLine TriviaTypeSpace ) + +type TriviaCollection []Trivia + +func (t TriviaCollection) Has(triviaType TriviaType) bool { + for _, trivia := range t { + if trivia.Type == triviaType { + return true + } + } + return false +} diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 6e5b64910a..e7c92c38b1 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -265,6 +265,24 @@ func (p *parser) next() { } } +func (p *parser) isFollowedByTrivia(triviaType lexer.TriviaType) bool { + if p.current.TrailingTrivia.Has(triviaType) { + return true + } + + cursor := p.tokens.Cursor() + previous := p.current + p.next() + + // The buffered tokens are replayed to allow them to be re-parsed. + defer func() { + p.tokens.Revert(cursor) + p.current = previous + }() + + return p.current.LeadingTrivia.Has(triviaType) +} + // nextSemanticToken advances past the current token to the next semantic token. // It skips whitespace, including newlines, and comments func (p *parser) nextSemanticToken() { diff --git a/runtime/parser/type.go b/runtime/parser/type.go index a9df066c64..2ffdfaac38 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -486,20 +486,8 @@ func defineIntersectionOrDictionaryType() { lexer.TokenBraceOpen, func(p *parser, rightBindingPower int, left ast.Type) (result ast.Type, err error, done bool) { - // Perform a lookahead - - current := p.current - cursor := p.tokens.Cursor() - - // Skip the `{` token. - p.next() - // In case there is a space, the type is *not* considered a restricted type. - // The buffered tokens are replayed to allow them to be re-parsed. - if p.current.Is(lexer.TokenSpace) { - p.current = current - p.tokens.Revert(cursor) - + if p.isFollowedByTrivia(lexer.TriviaTypeSpace) { return left, nil, true } From e0d871484e7df369a4de50353446fa433a15abd8 Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 18:17:15 +0200 Subject: [PATCH 13/84] remove custom meta left denotations This was causing some issues when parsing conformations - `T{` (legacy restricted type) and `T {` (type followed by block start token) were treated the same, since we removed space tokens. --- runtime/parser/type.go | 38 +------------------------------------- 1 file changed, 1 insertion(+), 37 deletions(-) diff --git a/runtime/parser/type.go b/runtime/parser/type.go index 2ffdfaac38..24bbcc76d2 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -48,7 +48,6 @@ type typeMetaLeftDenotationFunc func( var typeLeftBindingPowers [lexer.TokenMax]int var typeLeftDenotations [lexer.TokenMax]typeLeftDenotationFunc -var typeMetaLeftDenotations [lexer.TokenMax]typeMetaLeftDenotationFunc func setTypeNullDenotation(tokenType lexer.TokenType, nullDenotation typeNullDenotationFunc) { current := typeNullDenotations[tokenType] @@ -80,17 +79,6 @@ func setTypeLeftDenotation(tokenType lexer.TokenType, leftDenotation typeLeftDen typeLeftDenotations[tokenType] = leftDenotation } -func setTypeMetaLeftDenotation(tokenType lexer.TokenType, metaLeftDenotation typeMetaLeftDenotationFunc) { - current := typeMetaLeftDenotations[tokenType] - if current != nil { - panic(errors.NewUnexpectedError( - "type meta left denotation for token %s already exists", - tokenType, - )) - } - typeMetaLeftDenotations[tokenType] = metaLeftDenotation -} - type prefixTypeFunc func(parser *parser, right ast.Type, tokenRange ast.Range) ast.Type type postfixTypeFunc func(parser *parser, left ast.Type, tokenRange ast.Range) ast.Type @@ -479,21 +467,6 @@ func defineIntersectionOrDictionaryType() { }, ) - // While restricted types have been removed from Cadence, during the first few months of the - // migration period, leave a special error in place to help developers - // TODO: remove this after Stable Cadence migration period is finished - setTypeMetaLeftDenotation( - lexer.TokenBraceOpen, - func(p *parser, rightBindingPower int, left ast.Type) (result ast.Type, err error, done bool) { - - // In case there is a space, the type is *not* considered a restricted type. - if p.isFollowedByTrivia(lexer.TriviaTypeSpace) { - return left, nil, true - } - - return nil, &RestrictedTypeError{Range: p.current.Range}, true - }, - ) } func parseNominalType( @@ -679,17 +652,8 @@ func applyTypeMetaLeftDenotation( ) { // By default, left denotations are applied if the right binding power // is less than the left binding power of the current token. - // - // Token-specific meta-left denotations allow customizing this, - // e.g. determining the left binding power based on parsing more tokens, - // or performing look-ahead - - metaLeftDenotation := typeMetaLeftDenotations[p.current.Type] - if metaLeftDenotation == nil { - metaLeftDenotation = defaultTypeMetaLeftDenotation - } - return metaLeftDenotation(p, rightBindingPower, left) + return defaultTypeMetaLeftDenotation(p, rightBindingPower, left) } // defaultTypeMetaLeftDenotation is the default type left denotation, which applies From 8fe1b5f3772373eb67980f3693c6876f9b1d122d Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 18:30:01 +0200 Subject: [PATCH 14/84] keep emiting space/newline tokens to make the parsing logic work --- runtime/parser/lexer/lexer.go | 36 +++++++++-------------------------- runtime/parser/lexer/state.go | 8 -------- 2 files changed, 9 insertions(+), 35 deletions(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 9734b9c36e..06fd6f4f7d 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -340,40 +340,22 @@ func (l *lexer) emitTrivia(triviaType TriviaType) { Range: currentRange, }) - l.consume(endPos) -} - -const useLegacyTrivia = false - -// TODO(preserve-comments): Remove after refactoring is complete -func (l *lexer) emitLegacyTrivia(legacyTriviaType TokenType) { - if !useLegacyTrivia { - return - } - - if legacyTriviaType == TokenSpace { - trivia := l.input[l.startOffset:l.endOffset] - - var containsNewline bool - for _, r := range trivia { - if r == '\n' { - containsNewline = true - } - } - + // TODO(preserve-comments): Decide if we should refactor parsing logic to trivia tokens or keep using space token only + // Parsing logic depends on space tokens to determine how to parse certain sentences. + if triviaType == TriviaTypeSpace || triviaType == TriviaTypeNewLine { l.emit( TokenSpace, - Space{ - ContainsNewline: containsNewline, - }, + Space{ContainsNewline: triviaType == TriviaTypeNewLine}, l.startPosition(), - true, + false, ) - } else { - l.emitType(legacyTriviaType) } + + l.consume(endPos) } +const useLegacyTrivia = false + // endPos pre-computed end-position by calling l.endPos() func (l *lexer) consume(endPos position) { l.startOffset = l.endOffset diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 315f7afa29..754642b26f 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -113,7 +113,6 @@ func rootState(l *lexer) stateFn { case ' ', '\t', '\r': return spaceState case '\n': - l.emitLegacyTrivia(TokenSpace) l.emitTrivia(TriviaTypeNewLine) return rootState case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': @@ -127,7 +126,6 @@ func rootState(l *lexer) stateFn { case '/': return lineCommentState case '*': - l.emitLegacyTrivia(TokenBlockCommentStart) return blockCommentState(l, 0) default: l.backupOne() @@ -269,7 +267,6 @@ func spaceState(l *lexer) stateFn { // TODO(preserve-comments): Do we need to track memory for other token types as well? common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - l.emitLegacyTrivia(TokenSpace) l.emitTrivia(TriviaTypeSpace) return rootState @@ -303,7 +300,6 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitLegacyTrivia(TokenLineComment) l.emitTrivia(TriviaTypeInlineComment) return rootState } @@ -325,9 +321,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { if l.acceptOne('*') { starOffset := l.endOffset l.endOffset = beforeSlashOffset - l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = starOffset - l.emitLegacyTrivia(TokenBlockCommentStart) return blockCommentState(l, nesting+1) } @@ -336,9 +330,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { if l.acceptOne('/') { slashOffset := l.endOffset l.endOffset = beforeStarOffset - l.emitLegacyTrivia(TokenBlockCommentContent) l.endOffset = slashOffset - l.emitLegacyTrivia(TokenBlockCommentEnd) return blockCommentState(l, nesting-1) } } From 213ae488e2ba34833c8d6f1b97bb31509457b55c Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 18:30:30 +0200 Subject: [PATCH 15/84] Revert "remove custom meta left denotations" This reverts commit e0d871484e7df369a4de50353446fa433a15abd8. --- runtime/parser/type.go | 38 +++++++++++++++++++++++++++++++++++++- 1 file changed, 37 insertions(+), 1 deletion(-) diff --git a/runtime/parser/type.go b/runtime/parser/type.go index 24bbcc76d2..2ffdfaac38 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -48,6 +48,7 @@ type typeMetaLeftDenotationFunc func( var typeLeftBindingPowers [lexer.TokenMax]int var typeLeftDenotations [lexer.TokenMax]typeLeftDenotationFunc +var typeMetaLeftDenotations [lexer.TokenMax]typeMetaLeftDenotationFunc func setTypeNullDenotation(tokenType lexer.TokenType, nullDenotation typeNullDenotationFunc) { current := typeNullDenotations[tokenType] @@ -79,6 +80,17 @@ func setTypeLeftDenotation(tokenType lexer.TokenType, leftDenotation typeLeftDen typeLeftDenotations[tokenType] = leftDenotation } +func setTypeMetaLeftDenotation(tokenType lexer.TokenType, metaLeftDenotation typeMetaLeftDenotationFunc) { + current := typeMetaLeftDenotations[tokenType] + if current != nil { + panic(errors.NewUnexpectedError( + "type meta left denotation for token %s already exists", + tokenType, + )) + } + typeMetaLeftDenotations[tokenType] = metaLeftDenotation +} + type prefixTypeFunc func(parser *parser, right ast.Type, tokenRange ast.Range) ast.Type type postfixTypeFunc func(parser *parser, left ast.Type, tokenRange ast.Range) ast.Type @@ -467,6 +479,21 @@ func defineIntersectionOrDictionaryType() { }, ) + // While restricted types have been removed from Cadence, during the first few months of the + // migration period, leave a special error in place to help developers + // TODO: remove this after Stable Cadence migration period is finished + setTypeMetaLeftDenotation( + lexer.TokenBraceOpen, + func(p *parser, rightBindingPower int, left ast.Type) (result ast.Type, err error, done bool) { + + // In case there is a space, the type is *not* considered a restricted type. + if p.isFollowedByTrivia(lexer.TriviaTypeSpace) { + return left, nil, true + } + + return nil, &RestrictedTypeError{Range: p.current.Range}, true + }, + ) } func parseNominalType( @@ -652,8 +679,17 @@ func applyTypeMetaLeftDenotation( ) { // By default, left denotations are applied if the right binding power // is less than the left binding power of the current token. + // + // Token-specific meta-left denotations allow customizing this, + // e.g. determining the left binding power based on parsing more tokens, + // or performing look-ahead + + metaLeftDenotation := typeMetaLeftDenotations[p.current.Type] + if metaLeftDenotation == nil { + metaLeftDenotation = defaultTypeMetaLeftDenotation + } - return defaultTypeMetaLeftDenotation(p, rightBindingPower, left) + return metaLeftDenotation(p, rightBindingPower, left) } // defaultTypeMetaLeftDenotation is the default type left denotation, which applies From 2d7093ddc1389749a3a1fac0435ace6cacf338ad Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 18:30:35 +0200 Subject: [PATCH 16/84] Revert "introduce `isFollowedByTrivia`" This reverts commit 4158ce86ede111c8cb0efddc03ad9493937af568. --- runtime/parser/lexer/token.go | 4 ++-- runtime/parser/lexer/trivia.go | 11 ----------- runtime/parser/parser.go | 18 ------------------ runtime/parser/type.go | 14 +++++++++++++- 4 files changed, 15 insertions(+), 32 deletions(-) diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 6b573619a4..9982a3d616 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -28,9 +28,9 @@ type Token struct { ast.Range Type TokenType // leading Trivia up to and including the first contiguous sequence of newlines characters. - LeadingTrivia TriviaCollection + LeadingTrivia []Trivia // trailing Trivia up to, but not including, the next newline character. - TrailingTrivia TriviaCollection + TrailingTrivia []Trivia } func (t Token) Is(ty TokenType) bool { diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index 4fd702098b..4eb0dfa0e9 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -17,14 +17,3 @@ const ( TriviaTypeNewLine TriviaTypeSpace ) - -type TriviaCollection []Trivia - -func (t TriviaCollection) Has(triviaType TriviaType) bool { - for _, trivia := range t { - if trivia.Type == triviaType { - return true - } - } - return false -} diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index e7c92c38b1..6e5b64910a 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -265,24 +265,6 @@ func (p *parser) next() { } } -func (p *parser) isFollowedByTrivia(triviaType lexer.TriviaType) bool { - if p.current.TrailingTrivia.Has(triviaType) { - return true - } - - cursor := p.tokens.Cursor() - previous := p.current - p.next() - - // The buffered tokens are replayed to allow them to be re-parsed. - defer func() { - p.tokens.Revert(cursor) - p.current = previous - }() - - return p.current.LeadingTrivia.Has(triviaType) -} - // nextSemanticToken advances past the current token to the next semantic token. // It skips whitespace, including newlines, and comments func (p *parser) nextSemanticToken() { diff --git a/runtime/parser/type.go b/runtime/parser/type.go index 2ffdfaac38..a9df066c64 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -486,8 +486,20 @@ func defineIntersectionOrDictionaryType() { lexer.TokenBraceOpen, func(p *parser, rightBindingPower int, left ast.Type) (result ast.Type, err error, done bool) { + // Perform a lookahead + + current := p.current + cursor := p.tokens.Cursor() + + // Skip the `{` token. + p.next() + // In case there is a space, the type is *not* considered a restricted type. - if p.isFollowedByTrivia(lexer.TriviaTypeSpace) { + // The buffered tokens are replayed to allow them to be re-parsed. + if p.current.Is(lexer.TokenSpace) { + p.current = current + p.tokens.Revert(cursor) + return left, nil, true } From 4c106aaad232c2486c13b450d46d8744e6d03833 Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 18:45:18 +0200 Subject: [PATCH 17/84] remove comment parsing logic and token types --- runtime/old_parser/comment.go | 64 ----------------------- runtime/old_parser/expression_test.go | 66 ------------------------ runtime/old_parser/parser.go | 43 +--------------- runtime/parser/comment.go | 64 ----------------------- runtime/parser/declaration.go | 65 ++++++++++++------------ runtime/parser/expression.go | 30 +++++------ runtime/parser/expression_test.go | 66 ------------------------ runtime/parser/function.go | 18 +++---- runtime/parser/lexer/tokentype.go | 13 ----- runtime/parser/parser.go | 73 ++++----------------------- runtime/parser/parser_test.go | 2 +- runtime/parser/statement.go | 36 ++++++------- runtime/parser/transaction.go | 15 +++--- runtime/parser/type.go | 38 +++++++------- 14 files changed, 116 insertions(+), 477 deletions(-) delete mode 100644 runtime/old_parser/comment.go delete mode 100644 runtime/parser/comment.go diff --git a/runtime/old_parser/comment.go b/runtime/old_parser/comment.go deleted file mode 100644 index 766abdd135..0000000000 --- a/runtime/old_parser/comment.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package old_parser - -import ( - "github.com/onflow/cadence/runtime/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/runtime/old_parser/expression_test.go b/runtime/old_parser/expression_test.go index f1fb98ea73..25af7d15ce 100644 --- a/runtime/old_parser/expression_test.go +++ b/runtime/old_parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" - "github.com/onflow/cadence/runtime/parser/lexer" "github.com/onflow/cadence/runtime/tests/utils" ) @@ -2134,71 +2133,6 @@ func TestParseBlockComment(t *testing.T) { ) }) - t.Run("invalid content", func(t *testing.T) { - - t.Parallel() - - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 0, - Column: 0, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 1, - Column: 1, - }, - }, - }, - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 4, - Column: 4, - }, - }, - }, - {Type: lexer.TokenEOF}, - }, - input: []byte(`/*foo`), - } - - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, - ) - utils.AssertEqualWithDiff(t, - []error{ - &SyntaxError{ - Message: "unexpected token identifier in block comment", - Pos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - }, - }, - errs, - ) - }) } func BenchmarkParseInfix(b *testing.B) { diff --git a/runtime/old_parser/parser.go b/runtime/old_parser/parser.go index 299109886a..c0aed523c2 100644 --- a/runtime/old_parser/parser.go +++ b/runtime/old_parser/parser.go @@ -19,7 +19,6 @@ package old_parser import ( - "bytes" "os" "strings" @@ -425,9 +424,6 @@ func (p *parser) skipSpaceAndComments() (containsNewline bool) { return } -var blockCommentDocStringPrefix = []byte("/**") -var lineCommentDocStringPrefix = []byte("///") - func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { var docStringBuilder strings.Builder defer func() { @@ -436,7 +432,7 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt } }() - var atEnd, insideLineDocString bool + var atEnd bool for !atEnd { switch p.current.Type { @@ -457,43 +453,6 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() - case lexer.TokenBlockCommentStart: - commentStartOffset := p.current.StartPos.Offset - endToken, ok := p.parseBlockComment() - - if ok && options.parseDocStrings { - commentEndOffset := endToken.EndPos.Offset - - contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] - - insideLineDocString = false - docStringBuilder.Reset() - if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { - // Strip prefix (`/**`) - docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) - } - } - - case lexer.TokenLineComment: - if options.parseDocStrings { - comment := p.currentTokenSource() - if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { - if insideLineDocString { - docStringBuilder.WriteByte('\n') - } else { - insideLineDocString = true - docStringBuilder.Reset() - } - // Strip prefix - docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) - } else { - insideLineDocString = false - docStringBuilder.Reset() - } - } - - p.next() - default: atEnd = true } diff --git a/runtime/parser/comment.go b/runtime/parser/comment.go deleted file mode 100644 index 1ca171889e..0000000000 --- a/runtime/parser/comment.go +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package parser - -import ( - "github.com/onflow/cadence/runtime/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 2c9775ae18..ce7cd94579 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -32,9 +32,10 @@ import ( func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations []ast.Declaration, err error) { for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + // TODO(preserve-comments): Compute doc string + var docString string + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -77,7 +78,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { nativeModifierEnabled := p.config.NativeModifierEnabled for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenPragma: @@ -373,7 +374,7 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() entitlements := []*ast.NominalType{firstTy} var separator lexer.TokenType @@ -432,7 +433,7 @@ func parseAccess(p *parser) (ast.Access, error) { return ast.AccessNotSpecified, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return ast.AccessNotSpecified, p.syntaxError( @@ -478,7 +479,7 @@ func parseAccess(p *parser) (ast.Access, error) { } access = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) @@ -545,7 +546,7 @@ func parseVariableDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() transfer := parseTransfer(p) if transfer == nil { return nil, p.syntaxError("expected transfer") @@ -556,7 +557,7 @@ func parseVariableDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() secondTransfer := parseTransfer(p) var secondValue ast.Expression @@ -1057,7 +1058,7 @@ func parseFieldWithVariableKind( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() typeAnnotation, err := parseTypeAnnotation(p) if err != nil { @@ -1097,14 +1098,14 @@ func parseEntitlementMapping(p *parser, docString string) (*ast.EntitlementMapRe ) } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenRightArrow) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() outputType, err := parseType(p, lowestBindingPower) if err != nil { @@ -1119,7 +1120,7 @@ func parseEntitlementMapping(p *parser, docString string) (*ast.EntitlementMapRe ) } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewEntitlementMapRelation(p.memoryGauge, inputNominalType, outputNominalType), nil } @@ -1129,9 +1130,10 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy var elements []ast.EntitlementMapElement for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + // TODO(preserve-comments): Compute doc string + var docString string + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -1156,7 +1158,7 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy ) } - p.skipSpaceAndComments() + p.skipSpace() elements = append(elements, outputNominalType) } else { mapping, err := parseEntitlementMapping(p, docString) @@ -1227,7 +1229,7 @@ func parseEntitlementOrMappingDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1285,7 +1287,7 @@ func parseConformances(p *parser) ([]*ast.NominalType, error) { } } - p.skipSpaceAndComments() + p.skipSpace() return conformances, nil } @@ -1319,7 +1321,7 @@ func parseCompositeOrInterfaceDeclaration( var identifier ast.Identifier for { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( "expected %s, got %s", @@ -1355,7 +1357,7 @@ func parseCompositeOrInterfaceDeclaration( } } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1372,7 +1374,7 @@ func parseCompositeOrInterfaceDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1429,7 +1431,7 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.isToken(p.current, lexer.TokenIdentifier, KeywordFor) { return nil, p.syntaxError( @@ -1461,7 +1463,7 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() conformances, err := parseConformances(p) if err != nil { @@ -1473,14 +1475,14 @@ func parseAttachmentDeclaration( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() members, err := parseMembersAndNestedDeclarations(p, lexer.TokenBraceClose) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBraceClose) if err != nil { @@ -1514,9 +1516,10 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) var declarations []ast.Declaration for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + // TODO(preserve-comments): Compute doc string + var docString string + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { @@ -1573,7 +1576,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio nativeModifierEnabled := p.config.NativeModifierEnabled for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -1871,7 +1874,7 @@ func parseFieldDeclarationWithoutVariableKind( return nil, err } - p.skipSpaceAndComments() + p.skipSpace() typeAnnotation, err := parseTypeAnnotation(p) if err != nil { diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 60911b45c8..87db3ed912 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -597,7 +597,7 @@ func defineLessThanOrTypeArgumentsExpression() { return err } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken, err := p.mustOne(lexer.TokenParenOpen) if err != nil { return err @@ -832,7 +832,7 @@ func defineIdentifierExpression() { current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // skip the `fun` keyword @@ -965,7 +965,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() if !p.isToken(p.current, lexer.TokenIdentifier, KeywordTo) { return nil, p.syntaxError( @@ -982,7 +982,7 @@ func parseAttachExpressionRemainder(p *parser, token lexer.Token) (*ast.AttachEx return nil, err } - p.skipSpaceAndComments() + p.skipSpace() return ast.NewAttachExpression(p.memoryGauge, base, attachment, token.StartPos), nil } @@ -1017,7 +1017,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as atEnd := false expectArgument := true for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -1057,7 +1057,7 @@ func parseArgumentListRemainder(p *parser) (arguments []*ast.Argument, endPos as return nil, ast.EmptyPosition, err } - p.skipSpaceAndComments() + p.skipSpace() argument.TrailingSeparatorPos = p.current.StartPos @@ -1081,7 +1081,7 @@ func parseArgument(p *parser) (*ast.Argument, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() // If a colon follows the expression, the expression was our label. if p.current.Is(lexer.TokenColon) { @@ -1122,7 +1122,7 @@ func defineNestedExpression() { setExprNullDenotation( lexer.TokenParenOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() // special case: parse a Void literal `()` if p.current.Type == lexer.TokenParenClose { @@ -1148,11 +1148,11 @@ func defineArrayExpression() { setExprNullDenotation( lexer.TokenBracketOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var values []ast.Expression for !p.current.Is(lexer.TokenBracketClose) { - p.skipSpaceAndComments() + p.skipSpace() if len(values) > 0 { if !p.current.Is(lexer.TokenComma) { break @@ -1189,11 +1189,11 @@ func defineDictionaryExpression() { setExprNullDenotation( lexer.TokenBraceOpen, func(p *parser, startToken lexer.Token) (ast.Expression, error) { - p.skipSpaceAndComments() + p.skipSpace() var entries []ast.DictionaryEntry for !p.current.Is(lexer.TokenBraceClose) { - p.skipSpaceAndComments() + p.skipSpace() if len(entries) > 0 { if !p.current.Is(lexer.TokenComma) { break @@ -1370,7 +1370,7 @@ func parseMemberAccess(p *parser, token lexer.Token, left ast.Expression, option if p.current.Is(lexer.TokenSpace) { errorPos := p.current.StartPos - p.skipSpaceAndComments() + p.skipSpace() p.report(NewSyntaxError( errorPos, "invalid whitespace after %s", @@ -1438,7 +1438,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { p.expressionDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -1453,7 +1453,7 @@ func parseExpression(p *parser, rightBindingPower int) (ast.Expression, error) { // Some left denotations do not support newlines before them, // to avoid ambiguities and potential underhanded code - p.parseTrivia(triviaOptions{ + p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) diff --git a/runtime/parser/expression_test.go b/runtime/parser/expression_test.go index f93e0769d4..0349e3256f 100644 --- a/runtime/parser/expression_test.go +++ b/runtime/parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" - "github.com/onflow/cadence/runtime/parser/lexer" "github.com/onflow/cadence/runtime/tests/utils" ) @@ -2239,71 +2238,6 @@ func TestParseBlockComment(t *testing.T) { ) }) - t.Run("invalid content", func(t *testing.T) { - - t.Parallel() - - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 0, - Column: 0, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 1, - Column: 1, - }, - }, - }, - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - EndPos: ast.Position{ - Line: 1, - Offset: 4, - Column: 4, - }, - }, - }, - {Type: lexer.TokenEOF}, - }, - input: []byte(`/*foo`), - } - - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, - ) - utils.AssertEqualWithDiff(t, - []error{ - &SyntaxError{ - Message: "unexpected token identifier in block comment", - Pos: ast.Position{ - Line: 1, - Offset: 2, - Column: 2, - }, - }, - }, - errs, - ) - }) } func TestParseMulInfixExpression(t *testing.T) { diff --git a/runtime/parser/function.go b/runtime/parser/function.go index e6e5cc0aae..1c3af038d7 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -35,7 +35,7 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { var parameters []*ast.Parameter - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenParenOpen) { return nil, p.syntaxError( @@ -55,7 +55,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: if !expectParameter { @@ -121,7 +121,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, error) { - p.skipSpaceAndComments() + p.skipSpace() startPos := p.current.StartPos @@ -167,7 +167,7 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var defaultArgument ast.Expression @@ -204,7 +204,7 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { var typeParameters []*ast.TypeParameter - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenLess) { return nil, nil @@ -220,7 +220,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: if !expectTypeParameter { @@ -286,7 +286,7 @@ func parseTypeParameterList(p *parser) (*ast.TypeParameterList, error) { } func parseTypeParameter(p *parser) (*ast.TypeParameter, error) { - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( @@ -397,7 +397,7 @@ func parseFunctionParameterListAndRest( current := p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenColon) { // Skip the colon p.nextSemanticToken() @@ -416,7 +416,7 @@ func parseFunctionParameterListAndRest( if functionBlockIsOptional { current = p.current cursor := p.tokens.Cursor() - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenBraceOpen) { p.tokens.Revert(cursor) p.current = current diff --git a/runtime/parser/lexer/tokentype.go b/runtime/parser/lexer/tokentype.go index dd5ced009c..0bdd92371c 100644 --- a/runtime/parser/lexer/tokentype.go +++ b/runtime/parser/lexer/tokentype.go @@ -69,11 +69,6 @@ const ( TokenEqualEqual TokenExclamationMark TokenNotEqual - // TODO(preserve-comments): Deprecate Trivia token types - TokenBlockCommentStart - TokenBlockCommentEnd - TokenBlockCommentContent - TokenLineComment TokenAmpersand TokenAmpersandAmpersand TokenCaret @@ -180,14 +175,6 @@ func (t TokenType) String() string { return `'!'` case TokenNotEqual: return `'!='` - case TokenBlockCommentStart: - return `'/*'` - case TokenBlockCommentContent: - return "block comment" - case TokenLineComment: - return "line comment" - case TokenBlockCommentEnd: - return `'*/'` case TokenAmpersand: return `'&'` case TokenAmpersandAmpersand: diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 6e5b64910a..6d6a5e4fd8 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -19,14 +19,11 @@ package parser import ( - "bytes" - "os" - "strings" - "github.com/onflow/cadence/runtime/ast" "github.com/onflow/cadence/runtime/common" "github.com/onflow/cadence/runtime/errors" "github.com/onflow/cadence/runtime/parser/lexer" + "os" ) // expressionDepthLimit is the limit of how deeply nested an expression can get @@ -182,7 +179,7 @@ func ParseTokenStream[T any]( return result, p.errors } - p.skipSpaceAndComments() + p.skipSpace() if !p.current.Is(lexer.TokenEOF) { p.reportSyntaxError("unexpected token: %s", p.current.Type) @@ -269,7 +266,7 @@ func (p *parser) next() { // It skips whitespace, including newlines, and comments func (p *parser) nextSemanticToken() { p.next() - p.skipSpaceAndComments() + p.skipSpace() } func (p *parser) mustOne(tokenType lexer.TokenType) (lexer.Token, error) { @@ -424,31 +421,20 @@ func (p *parser) replayBuffered() error { return nil } -type triviaOptions struct { - skipNewlines bool - parseDocStrings bool +type skipSpaceOptions struct { + skipNewlines bool } -// skipSpaceAndComments skips whitespace, including newlines, and comments -func (p *parser) skipSpaceAndComments() (containsNewline bool) { - containsNewline, _ = p.parseTrivia(triviaOptions{ +// skipSpace skips whitespace, including newlines, and comments +func (p *parser) skipSpace() (containsNewline bool) { + containsNewline = p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) return } -var blockCommentDocStringPrefix = []byte("/**") -var lineCommentDocStringPrefix = []byte("///") - -func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { - var docStringBuilder strings.Builder - defer func() { - if options.parseDocStrings { - docString = docStringBuilder.String() - } - }() - - var atEnd, insideLineDocString bool +func (p *parser) skipSpaceWithOptions(options skipSpaceOptions) (containsNewline bool) { + var atEnd bool for !atEnd { switch p.current.Type { @@ -469,43 +455,6 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() - case lexer.TokenBlockCommentStart: - commentStartOffset := p.current.StartPos.Offset - endToken, ok := p.parseBlockComment() - - if ok && options.parseDocStrings { - commentEndOffset := endToken.EndPos.Offset - - contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] - - insideLineDocString = false - docStringBuilder.Reset() - if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { - // Strip prefix (`/**`) - docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) - } - } - - case lexer.TokenLineComment: - if options.parseDocStrings { - comment := p.currentTokenSource() - if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { - if insideLineDocString { - docStringBuilder.WriteByte('\n') - } else { - insideLineDocString = true - docStringBuilder.Reset() - } - // Strip prefix - docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) - } else { - insideLineDocString = false - docStringBuilder.Reset() - } - } - - p.next() - default: atEnd = true } @@ -693,7 +642,7 @@ func ParseArgumentList( memoryGauge, input, func(p *parser) (ast.Arguments, error) { - p.skipSpaceAndComments() + p.skipSpace() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index a81ffaa1af..a01577d5e5 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -605,7 +605,7 @@ func TestParseEOF(t *testing.T) { if err != nil { return struct{}{}, err } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustToken(lexer.TokenIdentifier, "b") if err != nil { return struct{}{}, err diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index 35deb28838..e46f4ce303 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -27,7 +27,7 @@ import ( func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statements []ast.Statement, err error) { sawSemicolon := false for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: sawSemicolon = true @@ -71,7 +71,7 @@ func parseStatements(p *parser, isEndToken func(token lexer.Token) bool) (statem } func parseStatement(p *parser) (ast.Statement, error) { - p.skipSpaceAndComments() + p.skipSpace() // Flag for cases where we can tell early-on that the current token isn't being used as a keyword // e.g. soft keywords like `view` @@ -148,7 +148,7 @@ func parseStatement(p *parser) (ast.Statement, error) { // If the expression is followed by a transfer, // it is actually the target of an assignment or swap statement - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenEqual, lexer.TokenLeftArrow, lexer.TokenLeftArrowExclamation: transfer := parseTransfer(p) @@ -251,7 +251,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { endPosition := tokenRange.EndPos p.next() - sawNewLine, _ := p.parseTrivia(triviaOptions{ + sawNewLine := p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: false, }) @@ -336,7 +336,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { parseNested := false - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordElse) { p.nextSemanticToken() if p.isToken(p.current, lexer.TokenIdentifier, KeywordIf) { @@ -431,7 +431,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var index *ast.Identifier var identifier ast.Identifier @@ -444,7 +444,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() } else { identifier = firstValue } @@ -509,14 +509,14 @@ func parseBlock(p *parser) (*ast.Block, error) { } func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { - p.skipSpaceAndComments() + p.skipSpace() startToken, err := p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var preConditions *ast.Conditions if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { @@ -529,7 +529,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { preConditions = &conditions } - p.skipSpaceAndComments() + p.skipSpace() var postConditions *ast.Conditions if p.isToken(p.current, lexer.TokenIdentifier, KeywordPost) { @@ -574,7 +574,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { // parseConditions parses conditions (pre/post) func parseConditions(p *parser) (conditions ast.Conditions, err error) { - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err @@ -582,7 +582,7 @@ func parseConditions(p *parser) (conditions ast.Conditions, err error) { var done bool for !done { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenSemicolon: p.next() @@ -602,7 +602,7 @@ func parseConditions(p *parser) (conditions ast.Conditions, err error) { } } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceClose) if err != nil { return nil, err @@ -633,7 +633,7 @@ func parseCondition(p *parser) (ast.Condition, error) { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var message ast.Expression if p.current.Is(lexer.TokenColon) { @@ -718,7 +718,7 @@ func parseSwitchCases(p *parser) (cases []*ast.SwitchCase, err error) { } for { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -772,7 +772,7 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { return nil, err } } else { - p.skipSpaceAndComments() + p.skipSpace() } colonPos := p.current.StartPos @@ -832,7 +832,7 @@ func parseRemoveStatement( startPos := p.current.StartPos p.next() - p.skipSpaceAndComments() + p.skipSpace() attachment, err := parseType(p, lowestBindingPower) if err != nil { @@ -847,7 +847,7 @@ func parseRemoveStatement( ) } - p.skipSpaceAndComments() + p.skipSpace() // check and skip `from` keyword if !p.isToken(p.current, lexer.TokenIdentifier, KeywordFrom) { diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index 22b2b46184..8b3e9995af 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -58,7 +58,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD } } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { return nil, err @@ -76,7 +76,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var prepare *ast.SpecialFunctionDeclaration var execute *ast.SpecialFunctionDeclaration - p.skipSpaceAndComments() + p.skipSpace() if p.current.Is(lexer.TokenIdentifier) { keyword := p.currentTokenSource() @@ -123,7 +123,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var preConditions *ast.Conditions if execute == nil { - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordPre) { // Skip the `pre` keyword p.next() @@ -145,7 +145,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD sawPost := false atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenIdentifier: @@ -215,9 +215,10 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err error) { for { - _, docString := p.parseTrivia(triviaOptions{ - skipNewlines: true, - parseDocStrings: true, + // TODO(preserve-comments): Compute doc string + var docString string + p.skipSpaceWithOptions(skipSpaceOptions{ + skipNewlines: true, }) switch p.current.Type { diff --git a/runtime/parser/type.go b/runtime/parser/type.go index a9df066c64..d44467c7aa 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -155,12 +155,12 @@ func init() { func defineParenthesizedTypes() { setTypeNullDenotation(lexer.TokenParenOpen, func(p *parser, token lexer.Token) (ast.Type, error) { - p.skipSpaceAndComments() + p.skipSpace() innerType, err := parseType(p, lowestBindingPower) if err != nil { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenParenClose) return innerType, err }) @@ -212,7 +212,7 @@ func defineArrayType() { return nil, err } - p.skipSpaceAndComments() + p.skipSpace() var size *ast.IntegerExpression @@ -241,7 +241,7 @@ func defineArrayType() { } } - p.skipSpaceAndComments() + p.skipSpace() endToken, err := p.mustOne(lexer.TokenBracketClose) if err != nil { @@ -341,7 +341,7 @@ func defineIntersectionOrDictionaryType() { expectType := true for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -537,7 +537,7 @@ func parseNominalTypes( expectType := true atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case separator: @@ -588,7 +588,7 @@ func parseNominalTypes( func parseParameterTypeAnnotations(p *parser) (typeAnnotations []*ast.TypeAnnotation, err error) { - p.skipSpaceAndComments() + p.skipSpace() _, err = p.mustOne(lexer.TokenParenOpen) if err != nil { return @@ -598,7 +598,7 @@ func parseParameterTypeAnnotations(p *parser) (typeAnnotations []*ast.TypeAnnota atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: if expectTypeAnnotation { @@ -656,7 +656,7 @@ func parseType(p *parser, rightBindingPower int) (ast.Type, error) { p.typeDepth-- }() - p.skipSpaceAndComments() + p.skipSpace() t := p.current p.next() @@ -729,7 +729,7 @@ func defaultTypeMetaLeftDenotation( } func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { - p.skipSpaceAndComments() + p.skipSpace() startPos := p.current.StartPos @@ -771,7 +771,7 @@ func applyTypeLeftDenotation(p *parser, token lexer.Token, left ast.Type) (ast.T } func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, error) { - p.skipSpaceAndComments() + p.skipSpace() identifier, err := p.mustOne(lexer.TokenIdentifier) if err != nil { return nil, err @@ -782,7 +782,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, return nil, err } - p.skipSpaceAndComments() + p.skipSpace() parenOpenToken, err := p.mustOne(lexer.TokenParenOpen) if err != nil { return nil, err @@ -830,7 +830,7 @@ func parseCommaSeparatedTypeAnnotations( expectTypeAnnotation := true atEnd := false for !atEnd { - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -912,14 +912,14 @@ func defineIdentifierTypes() { func(p *parser, token lexer.Token) (ast.Type, error) { switch string(p.tokenSource(token)) { case KeywordAuth: - p.skipSpaceAndComments() + p.skipSpace() var authorization ast.Authorization if p.current.Is(lexer.TokenParenOpen) { p.next() - p.skipSpaceAndComments() + p.skipSpace() var err error authorization, err = parseAuthorization(p) @@ -930,7 +930,7 @@ func defineIdentifierTypes() { p.reportSyntaxError("expected authorization (entitlement list)") } - p.skipSpaceAndComments() + p.skipSpace() _, err := p.mustOne(lexer.TokenAmpersand) if err != nil { @@ -950,7 +950,7 @@ func defineIdentifierTypes() { ), nil case KeywordFun: - p.skipSpaceAndComments() + p.skipSpace() return parseFunctionType(p, token.StartPos, ast.FunctionPurityUnspecified) case KeywordView: @@ -959,7 +959,7 @@ func defineIdentifierTypes() { cursor := p.tokens.Cursor() // look ahead for the `fun` keyword, if it exists - p.skipSpaceAndComments() + p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { // skip the `fun` keyword @@ -990,7 +990,7 @@ func parseAuthorization(p *parser) (auth ast.Authorization, err error) { return nil, err } auth = ast.NewMappedAccess(entitlementMapName, keywordPos) - p.skipSpaceAndComments() + p.skipSpace() default: entitlements, err := parseEntitlementList(p) From ae6525c0f8244cc4fef18ddb1be5f392c90072a6 Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 21:43:23 +0200 Subject: [PATCH 18/84] combine space and newline trivia types --- runtime/parser/lexer/lexer.go | 18 ++++++++----- runtime/parser/lexer/lexer_test.go | 20 +++++++------- runtime/parser/lexer/state.go | 43 +++++++++++++++++------------- runtime/parser/lexer/trivia.go | 4 +-- 4 files changed, 48 insertions(+), 37 deletions(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 06fd6f4f7d..966836d642 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -303,7 +303,7 @@ func (l *lexer) updatePreviousTrailingTrivia() { var leadingTrivia []Trivia trailingTriviaEnded := false for _, trivia := range l.currentTrivia { - if trivia.Type == TriviaTypeNewLine { + if trivia.Type == TriviaTypeSpace && trivia.ContainsNewLine { trailingTriviaEnded = true } if trailingTriviaEnded { @@ -317,7 +317,7 @@ func (l *lexer) updatePreviousTrailingTrivia() { l.currentTrivia = leadingTrivia } -func (l *lexer) emitTrivia(triviaType TriviaType) { +func (l *lexer) emitTrivia(triviaType TriviaType, containsNewLine bool) { endPos := l.endPos() currentRange := ast.NewRange( @@ -336,16 +336,17 @@ func (l *lexer) emitTrivia(triviaType TriviaType) { } l.currentTrivia = append(l.currentTrivia, Trivia{ - Type: triviaType, - Range: currentRange, + Type: triviaType, + ContainsNewLine: containsNewLine, + Range: currentRange, }) // TODO(preserve-comments): Decide if we should refactor parsing logic to trivia tokens or keep using space token only // Parsing logic depends on space tokens to determine how to parse certain sentences. - if triviaType == TriviaTypeSpace || triviaType == TriviaTypeNewLine { + if triviaType == TriviaTypeSpace { l.emit( TokenSpace, - Space{ContainsNewline: triviaType == TriviaTypeNewLine}, + Space{ContainsNewline: containsNewLine}, l.startPosition(), false, ) @@ -421,13 +422,16 @@ func (l *lexer) emitError(err error) { l.emit(TokenError, err, rangeStart, false) } -func (l *lexer) scanSpace() { +func (l *lexer) scanSpace() (containsNewline bool) { // lookahead is already lexed. // parse more, if any l.acceptWhile(func(r rune) bool { switch r { case ' ', '\t', '\r': return true + case '\n': + containsNewline = true + return true default: return false } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index d46a87156e..916f77fb44 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -464,7 +464,7 @@ func TestLexBasic(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, @@ -490,7 +490,7 @@ func TestLexBasic(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, @@ -862,7 +862,7 @@ func TestLexBasic(t *testing.T) { }, }, { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{ Offset: 24, @@ -951,7 +951,7 @@ func TestLexBasic(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{ Offset: 76, @@ -1173,7 +1173,7 @@ func TestLexString(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, @@ -1209,7 +1209,7 @@ func TestLexString(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, @@ -1299,7 +1299,7 @@ func TestLexString(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, @@ -1324,7 +1324,7 @@ func TestLexBlockComment(t *testing.T) { { Token: Token{ Type: TokenError, - SpaceOrError: errors.New(`end of the file in a comment`), + SpaceOrError: errors.New(`missing comment end '*/'`), Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, @@ -2006,7 +2006,7 @@ func TestLexIntegerLiterals(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, @@ -2345,7 +2345,7 @@ func TestLexLineComment(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeNewLine, + Type: TriviaTypeSpace, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 754642b26f..22c6d2e78c 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -111,10 +111,9 @@ func rootState(l *lexer) stateFn { case '_': return identifierState case ' ', '\t', '\r': - return spaceState + return spaceState(false) case '\n': - l.emitTrivia(TriviaTypeNewLine) - return rootState + return spaceState(true) case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9': return numberState case '"': @@ -122,11 +121,10 @@ func rootState(l *lexer) stateFn { case '/': r = l.next() switch r { - // TODO(preserve-comments): Deprecate Trivia token types case '/': return lineCommentState case '*': - return blockCommentState(l, 0) + return blockCommentState(l, 0, false) default: l.backupOne() l.emitType(TokenSlash) @@ -261,15 +259,20 @@ type Space struct { ContainsNewline bool } -func spaceState(l *lexer) stateFn { - l.scanSpace() +func spaceState(startIsNewline bool) stateFn { + return func(l *lexer) stateFn { + containsNewline := l.scanSpace() + containsNewline = containsNewline || startIsNewline + + l.scanSpace() - // TODO(preserve-comments): Do we need to track memory for other token types as well? - common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) + // TODO(preserve-comments): Do we need to track memory for other token types as well? + common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - l.emitTrivia(TriviaTypeSpace) + l.emitTrivia(TriviaTypeSpace, containsNewline) - return rootState + return rootState + } } func identifierState(l *lexer) stateFn { @@ -300,13 +303,13 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitTrivia(TriviaTypeInlineComment) + l.emitTrivia(TriviaTypeInlineComment, false) return rootState } -func blockCommentState(l *lexer, nesting int) stateFn { +func blockCommentState(l *lexer, nesting int, containsNewLine bool) stateFn { if nesting < 0 { - l.emitTrivia(TriviaTypeMultiLineComment) + l.emitTrivia(TriviaTypeMultiLineComment, containsNewLine) return rootState } @@ -314,7 +317,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { r := l.next() switch r { case EOF: - l.emitError(fmt.Errorf("end of the file in a comment")) + l.emitError(fmt.Errorf("missing comment end '*/'")) return nil case '/': beforeSlashOffset := l.prevEndOffset @@ -322,7 +325,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { starOffset := l.endOffset l.endOffset = beforeSlashOffset l.endOffset = starOffset - return blockCommentState(l, nesting+1) + return blockCommentState(l, nesting+1, containsNewLine) } case '*': @@ -331,10 +334,14 @@ func blockCommentState(l *lexer, nesting int) stateFn { slashOffset := l.endOffset l.endOffset = beforeStarOffset l.endOffset = slashOffset - return blockCommentState(l, nesting-1) + return blockCommentState(l, nesting-1, containsNewLine) } + + case '\n': + // TODO(preserve-comments): Test if this is correctly tracked + containsNewLine = true } - return blockCommentState(l, nesting) + return blockCommentState(l, nesting, containsNewLine) } } diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index 4eb0dfa0e9..9b36ff0193 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -3,7 +3,8 @@ package lexer import "github.com/onflow/cadence/runtime/ast" type Trivia struct { - Type TriviaType + Type TriviaType + ContainsNewLine bool // Position within the source code (includes opening/closing comment characters in case of comment trivia type) ast.Range } @@ -14,6 +15,5 @@ const ( TriviaTypeUnknown TriviaType = iota TriviaTypeInlineComment TriviaTypeMultiLineComment - TriviaTypeNewLine TriviaTypeSpace ) From d1592dfdafc97ea56c61c9692c2a1b7e654405da Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 10 Aug 2024 22:17:17 +0200 Subject: [PATCH 19/84] do not track trivia on space tokens --- runtime/parser/declaration.go | 8 ++++---- runtime/parser/lexer/lexer.go | 27 +++++++++++++++++++++------ runtime/parser/lexer/lexer_test.go | 7 +++++++ runtime/parser/lexer/token.go | 7 ++++--- runtime/parser/lexer/trivia.go | 1 + runtime/parser/statement.go | 2 +- 6 files changed, 38 insertions(+), 14 deletions(-) diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index ce7cd94579..4df2d3f40c 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -32,8 +32,6 @@ import ( func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations []ast.Declaration, err error) { for { - // TODO(preserve-comments): Compute doc string - var docString string p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) @@ -49,7 +47,7 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] default: var declaration ast.Declaration - declaration, err = parseDeclaration(p, docString) + declaration, err = parseDeclaration(p) if err != nil { return } @@ -63,7 +61,9 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] } } -func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { +func parseDeclaration(p *parser) (ast.Declaration, error) { + // TODO(preserve-comments): Refactor & remove + var docString string var access ast.Access = ast.AccessNotSpecified var accessPos *ast.Position diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 966836d642..a0e7534f1c 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -263,6 +263,17 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co l.updatePreviousTrailingTrivia() + // Only track trivia for non-space tokens + leadingTrivia := []Trivia{} + shouldConsumeTrivia := ty != TokenSpace + if shouldConsumeTrivia { + leadingTrivia = l.currentTrivia + defer (func() { + // Mark as fully consumed + l.currentTrivia = []Trivia{} + })() + } + token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -276,14 +287,11 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), - LeadingTrivia: l.currentTrivia, + LeadingTrivia: leadingTrivia, // Trailing Trivia can't be determined, as it wasn't consumed yet at this point. TrailingTrivia: []Trivia{}, } - // Mark as fully consumed - l.currentTrivia = []Trivia{} - l.tokens = append(l.tokens, token) l.tokenCount = len(l.tokens) @@ -301,7 +309,9 @@ func (l *lexer) updatePreviousTrailingTrivia() { // and leading trivia of the next token. var trailingTrivia []Trivia var leadingTrivia []Trivia - trailingTriviaEnded := false + previousToken := l.tokens[l.tokenCount-1] + // Only track trivia for non-space tokens. + trailingTriviaEnded := previousToken.Type == TokenSpace for _, trivia := range l.currentTrivia { if trivia.Type == TriviaTypeSpace && trivia.ContainsNewLine { trailingTriviaEnded = true @@ -313,7 +323,12 @@ func (l *lexer) updatePreviousTrailingTrivia() { } } - l.tokens[l.tokenCount-1].TrailingTrivia = trailingTrivia + // Only track trivia for non-space tokens. + for i := l.tokenCount - 1; i >= 0; i-- { + if l.tokens[i].Type != TokenSpace { + l.tokens[i].TrailingTrivia = trailingTrivia + } + } l.currentTrivia = leadingTrivia } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 916f77fb44..6892f13901 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1002,6 +1002,13 @@ transaction ( a: Int, // First operand 2 // Second operand 1 b: Int // Second operand 2 + /* + Third operand 1 + */ + c: Int /* Third operand 2 + + */ + // End of arguments comment ) { // Logs the sum 1 log(a, b) // Logs the sum 2 diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 9982a3d616..5f577e5c27 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -23,13 +23,14 @@ import ( ) type Token struct { - // TODO(preserve-comments): Rename to Error error SpaceOrError any ast.Range Type TokenType - // leading Trivia up to and including the first contiguous sequence of newlines characters. + // LeadingTrivia up to and including the first contiguous sequence of newlines characters. + // Not tracked for space token, since those are usually ignored in the parser. LeadingTrivia []Trivia - // trailing Trivia up to, but not including, the next newline character. + // TrailingTrivia up to, but not including, the next newline character. + // Not tracked for space token, since those are usually ignored in the parser. TrailingTrivia []Trivia } diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index 9b36ff0193..b9d0453625 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -11,6 +11,7 @@ type Trivia struct { type TriviaType uint8 +// TODO(preserve-comments): Refactor to Comment, since we now still track spaces using tokens const ( TriviaTypeUnknown TriviaType = iota TriviaTypeInlineComment diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index e46f4ce303..1e8cbcbdf3 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -127,7 +127,7 @@ func parseStatement(p *parser) (ast.Statement, error) { if !tokenIsIdentifier { // If it is not a keyword for a statement, // it might start with a keyword for a declaration - declaration, err := parseDeclaration(p, "") + declaration, err := parseDeclaration(p) if err != nil { return nil, err } From 8d2ce863fff0bdc757175ac80a3b7c79c2cd84a5 Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 12 Aug 2024 11:35:40 +0200 Subject: [PATCH 20/84] track leading comment for special function declarations --- runtime/ast/function_declaration.go | 3 +-- runtime/old_parser/declaration.go | 1 - runtime/old_parser/function.go | 1 - runtime/parser/declaration.go | 35 +++++++++++++---------------- runtime/parser/declaration_test.go | 11 ++------- runtime/parser/function.go | 2 -- runtime/parser/transaction.go | 18 +++++++-------- 7 files changed, 27 insertions(+), 44 deletions(-) diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index 2614edba08..e9d384c50d 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -94,7 +94,6 @@ func NewFunctionDeclarationWithComments( returnTypeAnnotation *TypeAnnotation, functionBlock *FunctionBlock, startPos Position, - docString string, comments Comments, ) *FunctionDeclaration { decl := NewFunctionDeclaration( @@ -109,7 +108,7 @@ func NewFunctionDeclarationWithComments( returnTypeAnnotation, functionBlock, startPos, - docString, + "", ) decl.Comments = comments return decl diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index 05f92e5d7d..56747a48f9 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -100,7 +100,6 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { accessPos, staticPos, nativePos, - docString, ) case keywordImport: diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index 9e02144775..d830f957a3 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -341,7 +341,6 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - docString, p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 4df2d3f40c..edd72de036 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -114,7 +114,6 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { purityPos, staticPos, nativePos, - docString, ) case KeywordImport: @@ -1026,8 +1025,9 @@ func parseFieldWithVariableKind( accessPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - docString string, ) (*ast.FieldDeclaration, error) { + // TODO(preserve-comments): Implement + var docString string startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) @@ -1516,8 +1516,6 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) var declarations []ast.Declaration for { - // TODO(preserve-comments): Compute doc string - var docString string p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) @@ -1532,7 +1530,7 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) return ast.NewMembers(p.memoryGauge, declarations), nil default: - memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p, docString) + memberOrNestedDeclaration, err := parseMemberOrNestedDeclaration(p) if err != nil { return nil, err } @@ -1557,7 +1555,9 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | eventDeclaration // | enumCase // | pragmaDeclaration -func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaration, error) { +func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { + // TODO(preserve-comments): Implement + var docString string const functionBlockIsOptional = true @@ -1602,7 +1602,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio accessPos, staticPos, nativePos, - docString, ) case KeywordCase: @@ -1613,7 +1612,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio if err != nil { return nil, err } - return parseEnumCase(p, access, accessPos, docString) + return parseEnumCase(p, access, accessPos) case KeywordFun: return parseFunctionDeclaration( @@ -1625,7 +1624,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio purityPos, staticPos, nativePos, - docString, ) case KeywordEvent: @@ -1801,7 +1799,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio staticPos, nativePos, identifier, - docString, ) case lexer.TokenParenOpen: @@ -1809,7 +1806,6 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio return nil, p.syntaxError("unexpected %s", p.current.Type) } - identifier := p.tokenToIdentifier(*previousIdentifierToken) return parseSpecialFunctionDeclaration( p, functionBlockIsOptional, @@ -1819,8 +1815,7 @@ func parseMemberOrNestedDeclaration(p *parser, docString string) (ast.Declaratio purityPos, staticPos, nativePos, - identifier, - docString, + *previousIdentifierToken, ) } @@ -1864,8 +1859,9 @@ func parseFieldDeclarationWithoutVariableKind( staticPos *ast.Position, nativePos *ast.Position, identifier ast.Identifier, - docString string, ) (*ast.FieldDeclaration, error) { + // TODO(preserve-comments): Implement + var docString string startPos := ast.EarliestPosition(identifier.Pos, accessPos, staticPos, nativePos) @@ -1907,10 +1903,10 @@ func parseSpecialFunctionDeclaration( purityPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - identifier ast.Identifier, - docString string, + identifierToken lexer.Token, ) (*ast.SpecialFunctionDeclaration, error) { + identifier := p.tokenToIdentifier(identifierToken) startPos := ast.EarliestPosition(identifier.Pos, accessPos, purityPos, staticPos, nativePos) parameterList, returnTypeAnnotation, functionBlock, err := @@ -1949,7 +1945,7 @@ func parseSpecialFunctionDeclaration( return ast.NewSpecialFunctionDeclaration( p.memoryGauge, declarationKind, - ast.NewFunctionDeclaration( + ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, purity, @@ -1961,7 +1957,7 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - docString, + p.newCommentsFromTrivia(identifierToken.LeadingTrivia, []lexer.Trivia{}), ), ), nil } @@ -1973,8 +1969,9 @@ func parseEnumCase( p *parser, access ast.Access, accessPos *ast.Position, - docString string, ) (*ast.EnumCaseDeclaration, error) { + // TODO(preserve-comments): Implement + var docString string startPos := p.current.StartPos if accessPos != nil { diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 567a482eca..c26e6d9903 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -2788,7 +2788,6 @@ func TestParseFieldWithVariableKind(t *testing.T) { nil, nil, nil, - "", ) }, Config{}, @@ -2873,10 +2872,7 @@ func TestParseField(t *testing.T) { nil, []byte(input), func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) + return parseMemberOrNestedDeclaration(p) }, config, ) @@ -8530,10 +8526,7 @@ func TestParseNestedPragma(t *testing.T) { nil, []byte(input), func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration( - p, - "", - ) + return parseMemberOrNestedDeclaration(p) }, config, ) diff --git a/runtime/parser/function.go b/runtime/parser/function.go index 1c3af038d7..cbf188d6c2 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -326,7 +326,6 @@ func parseFunctionDeclaration( purityPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, - docString string, ) (*ast.FunctionDeclaration, error) { startToken := p.current startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) @@ -372,7 +371,6 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - docString, p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), ), nil } diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index 8b3e9995af..792d026f9d 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -41,7 +41,8 @@ import ( // '}' func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionDeclaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos // Skip the `transaction` keyword p.nextSemanticToken() @@ -83,7 +84,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD switch string(keyword) { case KeywordPrepare: - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current // Skip the `prepare` keyword p.next() prepare, err = parseSpecialFunctionDeclaration( @@ -95,8 +96,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD nil, nil, nil, - identifier, - "", + identifierToken, ) if err != nil { return nil, err @@ -215,8 +215,6 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err error) { for { - // TODO(preserve-comments): Compute doc string - var docString string p.skipSpaceWithOptions(skipSpaceOptions{ skipNewlines: true, }) @@ -239,7 +237,6 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro nil, nil, nil, - docString, ) if err != nil { return nil, err @@ -259,7 +256,8 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro } func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) { - identifier := p.tokenToIdentifier(p.current) + identifierToken := p.current + identifier := p.tokenToIdentifier(identifierToken) // Skip the `execute` keyword p.nextSemanticToken() @@ -272,7 +270,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) return ast.NewSpecialFunctionDeclaration( p.memoryGauge, common.DeclarationKindExecute, - ast.NewFunctionDeclaration( + ast.NewFunctionDeclarationWithComments( p.memoryGauge, ast.AccessNotSpecified, ast.FunctionPurityUnspecified, @@ -289,7 +287,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - "", + p.newCommentsFromTrivia(identifierToken.LeadingTrivia, []lexer.Trivia{}), ), ), nil } From de4f950039b6b50e63e04689c697ef2f782d2556 Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 12 Aug 2024 15:00:13 +0200 Subject: [PATCH 21/84] track only comments as trivia in lexer --- runtime/ast/comments.go | 13 +- runtime/parser/lexer/lexer.go | 109 +++--- runtime/parser/lexer/lexer_test.go | 553 ++++++++++++++++++++++++++++- runtime/parser/lexer/state.go | 29 +- runtime/parser/lexer/token.go | 1 + runtime/parser/lexer/trivia.go | 3 +- 6 files changed, 623 insertions(+), 85 deletions(-) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index 4081e074df..aeb4bf84cb 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -6,12 +6,13 @@ import ( ) type Comments struct { - Leading []Comment `json:"-"` - Trailing []Comment `json:"-"` + Leading []*Comment `json:"-"` + Trailing []*Comment `json:"-"` } type Comment struct { source []byte + Range } func NewComment(memoryGauge common.MemoryGauge, source []byte) Comment { @@ -21,6 +22,14 @@ func NewComment(memoryGauge common.MemoryGauge, source []byte) Comment { } } +func NewCommentV2(memoryGauge common.MemoryGauge, source []byte, r Range) *Comment { + // TODO(preserve-comments): Track memory usage + return &Comment{ + source: source, + Range: r, + } +} + var blockCommentDocStringPrefix = []byte("/**") var blockCommentStringPrefix = []byte("/*") var lineCommentDocStringPrefix = []byte("///") diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index a0e7534f1c..08b3b312f5 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -74,8 +74,10 @@ type lexer struct { prev rune // canBackup indicates whether stepping back is allowed canBackup bool - // currentTrivia stores the current leading or trailing Trivia - currentTrivia []Trivia + // currentComments stores the current leading and/or trailing comments + // nil is used as sentinel value to track newlines + currentComments []*ast.Comment + isInTrailingCommentState bool } var _ TokenStream = &lexer{} @@ -101,9 +103,10 @@ func (l *lexer) Next() Token { pos, pos, ), - LeadingTrivia: l.currentTrivia, - // EOF has no trailing Trivia - TrailingTrivia: []Trivia{}, + Comments: ast.Comments{ + Leading: l.currentComments, + Trailing: []*ast.Comment{}, + }, } } @@ -194,7 +197,7 @@ func (l *lexer) run(state stateFn) { state = state(l) } - l.updatePreviousTrailingTrivia() + l.updatePreviousTrailingComments() } // next decodes the next rune (UTF8 character) from the input string. @@ -261,16 +264,15 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos := l.endPos() - l.updatePreviousTrailingTrivia() - // Only track trivia for non-space tokens - leadingTrivia := []Trivia{} + var leadingComments []*ast.Comment shouldConsumeTrivia := ty != TokenSpace if shouldConsumeTrivia { - leadingTrivia = l.currentTrivia + l.updatePreviousTrailingComments() + leadingComments = l.currentComments defer (func() { // Mark as fully consumed - l.currentTrivia = []Trivia{} + l.currentComments = []*ast.Comment{} })() } @@ -287,9 +289,11 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), - LeadingTrivia: leadingTrivia, - // Trailing Trivia can't be determined, as it wasn't consumed yet at this point. - TrailingTrivia: []Trivia{}, + Comments: ast.Comments{ + Leading: leadingComments, + // Trailing comments can't be determined, as it wasn't consumed yet at this point. + Trailing: []*ast.Comment{}, + }, } l.tokens = append(l.tokens, token) @@ -300,39 +304,53 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co } } -func (l *lexer) updatePreviousTrailingTrivia() { +func (l *lexer) updatePreviousTrailingComments() { if l.tokenCount == 0 { return } - // Split the current trivia into trailing trivia of the previous token - // and leading trivia of the next token. - var trailingTrivia []Trivia - var leadingTrivia []Trivia - previousToken := l.tokens[l.tokenCount-1] - // Only track trivia for non-space tokens. - trailingTriviaEnded := previousToken.Type == TokenSpace - for _, trivia := range l.currentTrivia { - if trivia.Type == TriviaTypeSpace && trivia.ContainsNewLine { + lastNonSpaceTokenIndex := -1 + for i := l.tokenCount - 1; i >= 0; i-- { + if l.tokens[i].Type != TokenSpace { + lastNonSpaceTokenIndex = i + break + } + } + + // Split the current comment into trailing comment of the previous token + // and leading comment of the next token. + var trailing []*ast.Comment + var leading []*ast.Comment + + if lastNonSpaceTokenIndex == -1 { + l.isInTrailingCommentState = false + } + + trailingTriviaEnded := lastNonSpaceTokenIndex == -1 + for _, comment := range l.currentComments { + if comment == nil { trailingTriviaEnded = true + continue } if trailingTriviaEnded { - leadingTrivia = append(leadingTrivia, trivia) + leading = append(leading, comment) } else { - trailingTrivia = append(trailingTrivia, trivia) + trailing = append(trailing, comment) } } - // Only track trivia for non-space tokens. - for i := l.tokenCount - 1; i >= 0; i-- { - if l.tokens[i].Type != TokenSpace { - l.tokens[i].TrailingTrivia = trailingTrivia - } + l.currentComments = leading + + if lastNonSpaceTokenIndex != -1 { + l.tokens[lastNonSpaceTokenIndex].Trailing = append(l.tokens[lastNonSpaceTokenIndex].Trailing, trailing...) } - l.currentTrivia = leadingTrivia } -func (l *lexer) emitTrivia(triviaType TriviaType, containsNewLine bool) { +func (l *lexer) emitNewlineSentinelComment() { + l.currentComments = append(l.currentComments, nil) +} + +func (l *lexer) emitComment() { endPos := l.endPos() currentRange := ast.NewRange( @@ -346,32 +364,17 @@ func (l *lexer) emitTrivia(triviaType TriviaType, containsNewLine bool) { ), ) - if l.currentTrivia == nil { - l.currentTrivia = []Trivia{} + if l.currentComments == nil { + l.currentComments = []*ast.Comment{} } - l.currentTrivia = append(l.currentTrivia, Trivia{ - Type: triviaType, - ContainsNewLine: containsNewLine, - Range: currentRange, - }) - - // TODO(preserve-comments): Decide if we should refactor parsing logic to trivia tokens or keep using space token only - // Parsing logic depends on space tokens to determine how to parse certain sentences. - if triviaType == TriviaTypeSpace { - l.emit( - TokenSpace, - Space{ContainsNewline: containsNewLine}, - l.startPosition(), - false, - ) - } + l.currentComments = append(l.currentComments, + ast.NewCommentV2(l.memoryGauge, currentRange.Source(l.input), currentRange), + ) l.consume(endPos) } -const useLegacyTrivia = false - // endPos pre-computed end-position by calling l.endPos() func (l *lexer) consume(endPos position) { l.startOffset = l.endOffset diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 6892f13901..6bd99968d2 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -993,28 +993,547 @@ func TestLexTrivia(t *testing.T) { t.Parallel() - // TODO(preserve-comments): Trailing Trivia contains one character more than it should - t.Run("Trivia", func(t *testing.T) { + t.Run("Simple comments", func(t *testing.T) { + testLex(t, + `/* before brace open */ { /* after brace open */ // after brace open 2 +// before brace close 1 +// before brace close 2 +}`, + []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 23, + Line: 1, + Column: 23, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* before brace open */"), ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 22, + Line: 1, + Column: 22, + }, + }), + }, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* after brace open */"), ast.Range{ + StartPos: ast.Position{ + Offset: 26, + Line: 1, + Column: 26, + }, + EndPos: ast.Position{ + Offset: 47, + Line: 1, + Column: 47, + }, + }), + ast.NewCommentV2(nil, []byte("// after brace open 2"), ast.Range{ + StartPos: ast.Position{ + Offset: 49, + Line: 1, + Column: 49, + }, + EndPos: ast.Position{ + Offset: 69, + Line: 1, + Column: 69, + }, + }), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + EndPos: ast.Position{ + Offset: 24, + Line: 1, + Column: 24, + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 25, + Line: 1, + Column: 25, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 48, + Line: 1, + Column: 48, + }, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + EndPos: ast.Position{ + Offset: 70, + Line: 1, + Column: 70, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 94, + Line: 2, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + EndPos: ast.Position{ + Offset: 118, + Line: 3, + Column: 23, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("// before brace close 1"), ast.Range{ + StartPos: ast.Position{ + Offset: 71, + Line: 2, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 93, + Line: 2, + Column: 22, + }, + }), + ast.NewCommentV2(nil, []byte("// before brace close 2"), ast.Range{ + StartPos: ast.Position{ + Offset: 95, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 117, + Line: 3, + Column: 22, + }, + }), + }, + Trailing: []*ast.Comment{}, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 119, + Line: 4, + Column: 0, + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 120, + Line: 4, + Column: 1, + }, + }, + }, + }, + }, + ) + }) + + t.Run("Complex comments", func(t *testing.T) { testLex(t, ` -// This transaction calculates sum -transaction ( - // First operand 1 - a: Int, // First operand 2 - // Second operand 1 - b: Int // Second operand 2 +// Before transaction identifier +transaction /* After transaction identifier */ ( + // Before first arg + a: Int, // After first arg /* - Third operand 1 + Before second arg */ - c: Int /* Third operand 2 + b: Int /* After second arg - */ - // End of arguments comment -) { - // Logs the sum 1 - log(a, b) // Logs the sum 2 -} + */ // After second arg 2 + // Before paren close +) /* After paren close */ { /* After brace open */ } // After brace close -`, []token{}) // TODO(preserve-comments): Define expected output +`, []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("// Before transaction identifier"), ast.Range{}), + }, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* After transaction identifier */"), ast.Range{}), + }, + }, + }, + Source: "transaction", + }, + { + Token: Token{ + Type: TokenSpace, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenParenOpen, + }, + Source: "(", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("// Before first arg"), ast.Range{}), + }, + Trailing: []*ast.Comment{}, + }, + }, + Source: "a", + }, + { + Token: Token{ + Type: TokenColon, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenComma, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("// After first arg"), ast.Range{}), + }, + }, + }, + Source: ",", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/*\n\tBefore second arg\n\t*/"), ast.Range{}), + }, + Trailing: []*ast.Comment{}, + }, + }, + Source: "b", + }, + { + Token: Token{ + Type: TokenColon, + }, + Source: ":", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenIdentifier, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* After second arg\n\n\t*/"), ast.Range{}), + ast.NewCommentV2(nil, []byte("// After second arg 2"), ast.Range{}), + }, + }, + }, + Source: "Int", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n\t", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenParenClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("// Before paren close"), ast.Range{}), + }, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* After paren close */"), ast.Range{}), + }, + }, + }, + Source: ")", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceOpen, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* After brace open */"), ast.Range{}), + }, + }, + }, + Source: "{", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenBraceClose, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewCommentV2(nil, []byte("/* After brace close */"), ast.Range{}), + }, + }, + }, + Source: "}", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + }, + Source: "\n\n", + }, + { + Token: Token{ + Type: TokenEOF, + }, + }, + }) }) } diff --git a/runtime/parser/lexer/state.go b/runtime/parser/lexer/state.go index 22c6d2e78c..435bb69ec9 100644 --- a/runtime/parser/lexer/state.go +++ b/runtime/parser/lexer/state.go @@ -124,7 +124,7 @@ func rootState(l *lexer) stateFn { case '/': return lineCommentState case '*': - return blockCommentState(l, 0, false) + return blockCommentState(l, 0) default: l.backupOne() l.emitType(TokenSlash) @@ -269,7 +269,16 @@ func spaceState(startIsNewline bool) stateFn { // TODO(preserve-comments): Do we need to track memory for other token types as well? common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) - l.emitTrivia(TriviaTypeSpace, containsNewline) + if containsNewline { + l.emitNewlineSentinelComment() + } + + l.emit( + TokenSpace, + Space{ContainsNewline: containsNewline}, + l.startPosition(), + true, + ) return rootState } @@ -303,13 +312,13 @@ func stringState(l *lexer) stateFn { func lineCommentState(l *lexer) stateFn { l.scanLineComment() - l.emitTrivia(TriviaTypeInlineComment, false) + l.emitComment() return rootState } -func blockCommentState(l *lexer, nesting int, containsNewLine bool) stateFn { +func blockCommentState(l *lexer, nesting int) stateFn { if nesting < 0 { - l.emitTrivia(TriviaTypeMultiLineComment, containsNewLine) + l.emitComment() return rootState } @@ -325,7 +334,7 @@ func blockCommentState(l *lexer, nesting int, containsNewLine bool) stateFn { starOffset := l.endOffset l.endOffset = beforeSlashOffset l.endOffset = starOffset - return blockCommentState(l, nesting+1, containsNewLine) + return blockCommentState(l, nesting+1) } case '*': @@ -334,14 +343,12 @@ func blockCommentState(l *lexer, nesting int, containsNewLine bool) stateFn { slashOffset := l.endOffset l.endOffset = beforeStarOffset l.endOffset = slashOffset - return blockCommentState(l, nesting-1, containsNewLine) + return blockCommentState(l, nesting-1) } - case '\n': - // TODO(preserve-comments): Test if this is correctly tracked - containsNewLine = true + return blockCommentState(l, nesting) } - return blockCommentState(l, nesting, containsNewLine) + return blockCommentState(l, nesting) } } diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index 5f577e5c27..be31d69b6a 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -25,6 +25,7 @@ import ( type Token struct { SpaceOrError any ast.Range + ast.Comments Type TokenType // LeadingTrivia up to and including the first contiguous sequence of newlines characters. // Not tracked for space token, since those are usually ignored in the parser. diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go index b9d0453625..79f00a4c39 100644 --- a/runtime/parser/lexer/trivia.go +++ b/runtime/parser/lexer/trivia.go @@ -3,8 +3,7 @@ package lexer import "github.com/onflow/cadence/runtime/ast" type Trivia struct { - Type TriviaType - ContainsNewLine bool + Type TriviaType // Position within the source code (includes opening/closing comment characters in case of comment trivia type) ast.Range } From d98ab2154bd981f4bb840ea38cf657674a01a544 Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 12 Aug 2024 17:41:18 +0200 Subject: [PATCH 22/84] update tests --- runtime/ast/comments.go | 11 +- runtime/old_parser/function.go | 4 +- runtime/parser/declaration.go | 4 +- runtime/parser/declaration_test.go | 24 +- runtime/parser/function.go | 4 +- runtime/parser/lexer/lexer.go | 14 +- runtime/parser/lexer/lexer_test.go | 509 ++++++++++++++++++++++++----- runtime/parser/parser.go | 26 -- runtime/parser/parser_test.go | 16 +- runtime/parser/statement.go | 5 +- runtime/parser/transaction.go | 4 +- 11 files changed, 473 insertions(+), 148 deletions(-) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index aeb4bf84cb..f7d94d6540 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -12,21 +12,12 @@ type Comments struct { type Comment struct { source []byte - Range } -func NewComment(memoryGauge common.MemoryGauge, source []byte) Comment { - // TODO(preserve-comments): Track memory usage - return Comment{ - source: source, - } -} - -func NewCommentV2(memoryGauge common.MemoryGauge, source []byte, r Range) *Comment { +func NewComment(memoryGauge common.MemoryGauge, source []byte) *Comment { // TODO(preserve-comments): Track memory usage return &Comment{ source: source, - Range: r, } } diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index d830f957a3..5c110f9273 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -341,7 +341,9 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), + ast.Comments{ + Leading: startToken.Leading, + }, ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index edd72de036..2afa1c127d 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -1957,7 +1957,9 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - p.newCommentsFromTrivia(identifierToken.LeadingTrivia, []lexer.Trivia{}), + ast.Comments{ + Leading: identifierToken.Leading, + }, ), ), nil } diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index c26e6d9903..ada66107ea 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -882,7 +882,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// Test")), }, }, @@ -924,7 +924,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// First line")), ast.NewComment(nil, []byte("/// Second line")), }, @@ -967,7 +967,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), }, }, @@ -1382,8 +1382,8 @@ func TestParseFunctionDeclaration(t *testing.T) { PostConditions: (*ast.Conditions)(nil), }, Comments: ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{}, + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, }, Identifier: ast.Identifier{ Identifier: "foo", @@ -3524,8 +3524,8 @@ func TestParseCompositeDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{}, + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, }, Identifier: ast.Identifier{ Identifier: "getFoo", @@ -8903,7 +8903,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// noReturnNoBlock")), }, }, @@ -8922,7 +8922,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// returnNoBlock")), }, }, @@ -8951,7 +8951,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// returnAndBlock")), }, }, @@ -9030,7 +9030,7 @@ func TestParseMemberDocStrings(t *testing.T) { FunctionDeclaration: &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// unknown")), }, }, @@ -9052,7 +9052,7 @@ func TestParseMemberDocStrings(t *testing.T) { FunctionDeclaration: &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// initNoBlock")), }, }, diff --git a/runtime/parser/function.go b/runtime/parser/function.go index cbf188d6c2..1667aa3a25 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -371,7 +371,9 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - p.newCommentsFromTrivia(startToken.LeadingTrivia, []lexer.Trivia{}), + ast.Comments{ + Leading: startToken.Leading, + }, ), nil } diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index 08b3b312f5..ab6615068b 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -76,8 +76,7 @@ type lexer struct { canBackup bool // currentComments stores the current leading and/or trailing comments // nil is used as sentinel value to track newlines - currentComments []*ast.Comment - isInTrailingCommentState bool + currentComments []*ast.Comment } var _ TokenStream = &lexer{} @@ -322,10 +321,6 @@ func (l *lexer) updatePreviousTrailingComments() { var trailing []*ast.Comment var leading []*ast.Comment - if lastNonSpaceTokenIndex == -1 { - l.isInTrailingCommentState = false - } - trailingTriviaEnded := lastNonSpaceTokenIndex == -1 for _, comment := range l.currentComments { if comment == nil { @@ -342,7 +337,8 @@ func (l *lexer) updatePreviousTrailingComments() { l.currentComments = leading if lastNonSpaceTokenIndex != -1 { - l.tokens[lastNonSpaceTokenIndex].Trailing = append(l.tokens[lastNonSpaceTokenIndex].Trailing, trailing...) + lastNonSpaceToken := &l.tokens[lastNonSpaceTokenIndex] + lastNonSpaceToken.Trailing = append(lastNonSpaceToken.Trailing, trailing...) } } @@ -368,9 +364,7 @@ func (l *lexer) emitComment() { l.currentComments = []*ast.Comment{} } - l.currentComments = append(l.currentComments, - ast.NewCommentV2(l.memoryGauge, currentRange.Source(l.input), currentRange), - ) + l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, currentRange.Source(l.input))) l.consume(endPos) } diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 6bd99968d2..aee8b4d7e3 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -989,11 +989,11 @@ func TestLexBasic(t *testing.T) { }) } -func TestLexTrivia(t *testing.T) { +func TestLexComments(t *testing.T) { t.Parallel() - t.Run("Simple comments", func(t *testing.T) { + t.Run("simple", func(t *testing.T) { testLex(t, `/* before brace open */ { /* after brace open */ // after brace open 2 // before brace close 1 @@ -1024,44 +1024,11 @@ func TestLexTrivia(t *testing.T) { Type: TokenBraceOpen, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* before brace open */"), ast.Range{ - StartPos: ast.Position{ - Offset: 0, - Line: 1, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 22, - Line: 1, - Column: 22, - }, - }), + ast.NewComment(nil, []byte("/* before brace open */")), }, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* after brace open */"), ast.Range{ - StartPos: ast.Position{ - Offset: 26, - Line: 1, - Column: 26, - }, - EndPos: ast.Position{ - Offset: 47, - Line: 1, - Column: 47, - }, - }), - ast.NewCommentV2(nil, []byte("// after brace open 2"), ast.Range{ - StartPos: ast.Position{ - Offset: 49, - Line: 1, - Column: 49, - }, - EndPos: ast.Position{ - Offset: 69, - Line: 1, - Column: 69, - }, - }), + ast.NewComment(nil, []byte("/* after brace open */")), + ast.NewComment(nil, []byte("// after brace open 2")), }, }, Range: ast.Range{ @@ -1179,30 +1146,8 @@ func TestLexTrivia(t *testing.T) { Type: TokenBraceClose, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("// before brace close 1"), ast.Range{ - StartPos: ast.Position{ - Offset: 71, - Line: 2, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 93, - Line: 2, - Column: 22, - }, - }), - ast.NewCommentV2(nil, []byte("// before brace close 2"), ast.Range{ - StartPos: ast.Position{ - Offset: 95, - Line: 3, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 117, - Line: 3, - Column: 22, - }, - }), + ast.NewComment(nil, []byte("// before brace close 1")), + ast.NewComment(nil, []byte("// before brace close 2")), }, Trailing: []*ast.Comment{}, }, @@ -1242,7 +1187,7 @@ func TestLexTrivia(t *testing.T) { ) }) - t.Run("Complex comments", func(t *testing.T) { + t.Run("complex", func(t *testing.T) { testLex(t, ` // Before transaction identifier transaction /* After transaction identifier */ ( @@ -1262,6 +1207,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + }, }, Source: "\n", }, @@ -1269,6 +1226,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + EndPos: ast.Position{ + Offset: 33, + Line: 2, + Column: 32, + }, + }, }, Source: "\n", }, @@ -1277,10 +1246,22 @@ transaction /* After transaction identifier */ ( Type: TokenIdentifier, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("// Before transaction identifier"), ast.Range{}), + ast.NewComment(nil, []byte("// Before transaction identifier")), }, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* After transaction identifier */"), ast.Range{}), + ast.NewComment(nil, []byte("/* After transaction identifier */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 34, + Line: 3, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 44, + Line: 3, + Column: 10, }, }, }, @@ -1288,19 +1269,57 @@ transaction /* After transaction identifier */ ( }, { Token: Token{ - Type: TokenSpace, + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + EndPos: ast.Position{ + Offset: 45, + Line: 3, + Column: 11, + }, + }, }, Source: " ", }, { Token: Token{ - Type: TokenSpace, + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + EndPos: ast.Position{ + Offset: 80, + Line: 3, + Column: 46, + }, + }, }, Source: " ", }, { Token: Token{ Type: TokenParenOpen, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + EndPos: ast.Position{ + Offset: 81, + Line: 3, + Column: 47, + }, + }, }, Source: "(", }, @@ -1308,13 +1327,37 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 82, + Line: 3, + Column: 48, + }, + EndPos: ast.Position{ + Offset: 83, + Line: 4, + Column: 0, + }, + }, }, Source: "\n\t", }, { Token: Token{ Type: TokenSpace, - SpaceOrError: Space{ContainsNewline: false}, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 103, + Line: 4, + Column: 20, + }, + EndPos: ast.Position{ + Offset: 104, + Line: 5, + Column: 0, + }, + }, }, Source: "\n\t", }, @@ -1323,16 +1366,40 @@ transaction /* After transaction identifier */ ( Type: TokenIdentifier, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("// Before first arg"), ast.Range{}), + ast.NewComment(nil, []byte("// Before first arg")), }, Trailing: []*ast.Comment{}, }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 105, + Line: 5, + Column: 1, + }, + }, }, Source: "a", }, { Token: Token{ Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 106, + Line: 5, + Column: 2, + }, + }, }, Source: ":", }, @@ -1340,12 +1407,36 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 107, + Line: 5, + Column: 3, + }, + }, }, Source: " ", }, { Token: Token{ Type: TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 108, + Line: 5, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 110, + Line: 5, + Column: 6, + }, + }, }, Source: "Int", }, @@ -1355,7 +1446,19 @@ transaction /* After transaction identifier */ ( Comments: ast.Comments{ Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("// After first arg"), ast.Range{}), + ast.NewComment(nil, []byte("// After first arg ")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 111, + Line: 5, + Column: 7, }, }, }, @@ -1365,6 +1468,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + EndPos: ast.Position{ + Offset: 112, + Line: 5, + Column: 8, + }, + }, }, Source: " ", }, @@ -1372,6 +1487,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 132, + Line: 5, + Column: 28, + }, + EndPos: ast.Position{ + Offset: 133, + Line: 6, + Column: 0, + }, + }, }, Source: "\n\t", }, @@ -1379,6 +1506,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 159, + Line: 8, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 160, + Line: 9, + Column: 0, + }, + }, }, Source: "\n\t", }, @@ -1387,16 +1526,40 @@ transaction /* After transaction identifier */ ( Type: TokenIdentifier, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/*\n\tBefore second arg\n\t*/"), ast.Range{}), + ast.NewComment(nil, []byte("/*\n\tBefore second arg\n\t*/")), }, Trailing: []*ast.Comment{}, }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 161, + Line: 9, + Column: 1, + }, + }, }, Source: "b", }, { Token: Token{ Type: TokenColon, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + EndPos: ast.Position{ + Offset: 162, + Line: 9, + Column: 2, + }, + }, }, Source: ":", }, @@ -1404,6 +1567,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 163, + Line: 9, + Column: 3, + }, + }, }, Source: " ", }, @@ -1413,8 +1588,20 @@ transaction /* After transaction identifier */ ( Comments: ast.Comments{ Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* After second arg\n\n\t*/"), ast.Range{}), - ast.NewCommentV2(nil, []byte("// After second arg 2"), ast.Range{}), + ast.NewComment(nil, []byte("/* After second arg\n\n\t*/")), + ast.NewComment(nil, []byte("// After second arg 2")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 164, + Line: 9, + Column: 4, + }, + EndPos: ast.Position{ + Offset: 166, + Line: 9, + Column: 6, }, }, }, @@ -1424,6 +1611,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + EndPos: ast.Position{ + Offset: 167, + Line: 9, + Column: 7, + }, + }, }, Source: " ", }, @@ -1431,6 +1630,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + EndPos: ast.Position{ + Offset: 192, + Line: 11, + Column: 3, + }, + }, }, Source: " ", }, @@ -1438,6 +1649,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 214, + Line: 11, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 215, + Line: 12, + Column: 0, + }, + }, }, Source: "\n\t", }, @@ -1445,6 +1668,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + EndPos: ast.Position{ + Offset: 237, + Line: 12, + Column: 22, + }, + }, }, Source: "\n", }, @@ -1453,10 +1688,22 @@ transaction /* After transaction identifier */ ( Type: TokenParenClose, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("// Before paren close"), ast.Range{}), + ast.NewComment(nil, []byte("// Before paren close")), }, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* After paren close */"), ast.Range{}), + ast.NewComment(nil, []byte("/* After paren close */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 238, + Line: 13, + Column: 0, }, }, }, @@ -1466,6 +1713,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + EndPos: ast.Position{ + Offset: 239, + Line: 13, + Column: 1, + }, + }, }, Source: " ", }, @@ -1473,6 +1732,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 25, + }, + }, }, Source: " ", }, @@ -1482,7 +1753,19 @@ transaction /* After transaction identifier */ ( Comments: ast.Comments{ Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* After brace open */"), ast.Range{}), + ast.NewComment(nil, []byte("/* After brace open */")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, + }, + EndPos: ast.Position{ + Offset: 264, + Line: 13, + Column: 26, }, }, }, @@ -1492,6 +1775,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + EndPos: ast.Position{ + Offset: 265, + Line: 13, + Column: 27, + }, + }, }, Source: " ", }, @@ -1499,6 +1794,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + EndPos: ast.Position{ + Offset: 288, + Line: 13, + Column: 50, + }, + }, }, Source: " ", }, @@ -1508,7 +1815,19 @@ transaction /* After transaction identifier */ ( Comments: ast.Comments{ Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ - ast.NewCommentV2(nil, []byte("/* After brace close */"), ast.Range{}), + ast.NewComment(nil, []byte("// After brace close")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, + }, + EndPos: ast.Position{ + Offset: 289, + Line: 13, + Column: 51, }, }, }, @@ -1518,6 +1837,18 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: false}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + EndPos: ast.Position{ + Offset: 290, + Line: 13, + Column: 52, + }, + }, }, Source: " ", }, @@ -1525,12 +1856,36 @@ transaction /* After transaction identifier */ ( Token: Token{ Type: TokenSpace, SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 311, + Line: 13, + Column: 73, + }, + EndPos: ast.Position{ + Offset: 312, + Line: 14, + Column: 0, + }, + }, }, Source: "\n\n", }, { Token: Token{ Type: TokenEOF, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 313, + Line: 15, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 313, + Line: 15, + Column: 0, + }, + }, }, }, }) diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 6d6a5e4fd8..6b9bf89e8b 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -507,32 +507,6 @@ func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { ) } -func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { - comments := ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{}, - } - - for _, t := range leadingTrivia { - if isTriviaComment(t) { - comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) - } - } - - for _, t := range trailingTrivia { - if isTriviaComment(t) { - comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) - } - } - - return comments -} - -func isTriviaComment(t lexer.Trivia) bool { - // Other trivia types (space, newlines) are not needed in further stages of the compiler - return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment -} - func (p *parser) startAmbiguity() { if p.ambiguityLevel == 0 { p.localReplayedTokensCount = 0 diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index a01577d5e5..43bf934991 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1046,15 +1046,15 @@ Trailing multi-line comment of second assert.True(t, ok) assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) assert.Equal(t, ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// Inline doc 1 of first")), ast.NewComment(nil, []byte("/// Inline doc 2 of first")), }, - Trailing: []ast.Comment{}, + Trailing: []*ast.Comment{}, }, first.Comments) assert.Equal(t, ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("// Trailing inline comment of first")), }, }, first.FunctionBlock.Block.Comments) @@ -1063,15 +1063,15 @@ Trailing multi-line comment of second assert.True(t, ok) assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) assert.Equal(t, ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), }, - Trailing: []ast.Comment{}, + Trailing: []*ast.Comment{}, }, second.Comments) assert.Equal(t, ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), }, }, second.FunctionBlock.Block.Comments) diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index 1e8cbcbdf3..efb3542012 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -564,7 +564,10 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), - p.newCommentsFromTrivia(startToken.LeadingTrivia, endToken.TrailingTrivia), + ast.Comments{ + Leading: startToken.Leading, + Trailing: endToken.Trailing, + }, ), preConditions, postConditions, diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index 792d026f9d..d5ebfd23b6 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -287,7 +287,9 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - p.newCommentsFromTrivia(identifierToken.LeadingTrivia, []lexer.Trivia{}), + ast.Comments{ + Leading: identifierToken.Leading, + }, ), ), nil } From 05ed569b96d7559052d3bb3ccea1f7b526679306 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 13 Aug 2024 21:44:27 +0200 Subject: [PATCH 23/84] test eof comment --- runtime/parser/lexer/lexer_test.go | 33 ++++++++++++++++++++++++++---- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index aee8b4d7e3..08a06f2a67 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -1202,6 +1202,7 @@ transaction /* After transaction identifier */ ( // Before paren close ) /* After paren close */ { /* After brace open */ } // After brace close +// Before EOF `, []token{ { Token: Token{ @@ -1873,16 +1874,40 @@ transaction /* After transaction identifier */ ( }, { Token: Token{ - Type: TokenEOF, + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, Range: ast.Range{ StartPos: ast.Position{ - Offset: 313, + Offset: 326, Line: 15, - Column: 0, + Column: 13, }, EndPos: ast.Position{ - Offset: 313, + Offset: 326, Line: 15, + Column: 13, + }, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenEOF, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before EOF")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 327, + Line: 16, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 327, + Line: 16, Column: 0, }, }, From dcff58bc1750783ca35e4522c2d53df16c8d5b35 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 13 Aug 2024 22:02:51 +0200 Subject: [PATCH 24/84] fix parser tests --- runtime/parser/lexer/lexer.go | 3 +-- runtime/parser/parser_test.go | 8 ++------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/runtime/parser/lexer/lexer.go b/runtime/parser/lexer/lexer.go index ab6615068b..6fc4b3655c 100644 --- a/runtime/parser/lexer/lexer.go +++ b/runtime/parser/lexer/lexer.go @@ -103,8 +103,7 @@ func (l *lexer) Next() Token { pos, ), Comments: ast.Comments{ - Leading: l.currentComments, - Trailing: []*ast.Comment{}, + Leading: l.currentComments, }, } diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index 43bf934991..e9ccecd5b9 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -540,8 +540,8 @@ func TestParseBuffering(t *testing.T) { []error{ &RestrictedTypeError{ Range: ast.Range{ - StartPos: ast.Position{Offset: 138, Line: 4, Column: 55}, - EndPos: ast.Position{Offset: 139, Line: 4, Column: 56}, + StartPos: ast.Position{Offset: 145, Line: 4, Column: 62}, + EndPos: ast.Position{Offset: 145, Line: 4, Column: 62}, }, }, }, @@ -1050,10 +1050,8 @@ Trailing multi-line comment of second ast.NewComment(nil, []byte("/// Inline doc 1 of first")), ast.NewComment(nil, []byte("/// Inline doc 2 of first")), }, - Trailing: []*ast.Comment{}, }, first.Comments) assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("// Trailing inline comment of first")), }, @@ -1067,10 +1065,8 @@ Trailing multi-line comment of second ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), }, - Trailing: []*ast.Comment{}, }, second.Comments) assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{}, Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), }, From 4c7047ec10b549ddff0dcdf836aebe33d2751265 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 13 Aug 2024 22:14:59 +0200 Subject: [PATCH 25/84] Revert "fix existing lexer tests" This reverts commit e4a10735 --- runtime/parser/lexer/lexer_test.go | 760 ++++++++++++++--------------- 1 file changed, 380 insertions(+), 380 deletions(-) diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index 08a06f2a67..e5e1db5271 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -88,6 +88,19 @@ func TestLexBasic(t *testing.T) { testLex(t, " 01\t 10", []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -95,26 +108,21 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, + }, + Source: "01", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - }, - }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, }, - Source: "01", + Source: "\t ", }, { Token: Token{ @@ -207,18 +215,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - }, }, Source: "2", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenPlus, @@ -226,18 +238,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - }, }, Source: "+", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -255,18 +271,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - }, - }, - }, }, Source: ")", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenStar, @@ -274,18 +294,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - }, }, Source: "*", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -330,18 +354,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - }, }, Source: "2", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenMinus, @@ -349,18 +377,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - }, }, Source: "-", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -378,18 +410,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, EndPos: ast.Position{Line: 1, Column: 6, Offset: 6}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, - }, - }, - }, }, Source: ")", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + EndPos: ast.Position{Line: 1, Column: 7, Offset: 7}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenSlash, @@ -397,18 +433,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 8, Offset: 8}, EndPos: ast.Position{Line: 1, Column: 8, Offset: 8}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - }, }, Source: "/", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -443,18 +483,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - }, }, Source: "1", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, + }, + }, + Source: " \n ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -462,25 +506,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 2, Offset: 5}, EndPos: ast.Position{Line: 2, Column: 2, Offset: 5}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, - EndPos: ast.Position{Line: 2, Column: 1, Offset: 4}, - }, - }, - }, }, Source: "2", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -488,15 +529,6 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 0, Offset: 7}, EndPos: ast.Position{Line: 3, Column: 0, Offset: 7}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 3, Offset: 6}, - EndPos: ast.Position{Line: 2, Column: 3, Offset: 6}, - }, - }, - }, }, }, }, @@ -514,18 +546,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - }, }, Source: "1", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDoubleQuestionMark, @@ -533,18 +569,22 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - }, }, Source: "??", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenDecimalIntegerLiteral, @@ -834,7 +874,7 @@ func TestLexBasic(t *testing.T) { ) }) - t.Run("trivia", func(t *testing.T) { + t.Run("Trivia", func(t *testing.T) { testLex(t, "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", []token{ @@ -848,95 +888,29 @@ func TestLexBasic(t *testing.T) { LeadingTrivia: []Trivia{ { Type: TriviaTypeInlineComment, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 0, - Line: 1, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 23, - Line: 1, - Column: 23, - }, - }, + Text: []byte("// test is in next line "), }, { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 24, - Line: 1, - Column: 24, - }, - EndPos: ast.Position{ - Offset: 24, - Line: 1, - Column: 24, - }, - }, + Type: TriviaTypeNewLine, + Text: []byte("\n"), }, { Type: TriviaTypeMultiLineComment, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 25, - Line: 2, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 42, - Line: 2, - Column: 17, - }, - }, + Text: []byte("/* test is here */"), }, { Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 43, - Line: 2, - Column: 18, - }, - EndPos: ast.Position{ - Offset: 43, - Line: 2, - Column: 18, - }, - }, + Text: []byte(" "), }, }, TrailingTrivia: []Trivia{ { Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 48, - Line: 2, - Column: 23, - }, - EndPos: ast.Position{ - Offset: 48, - Line: 2, - Column: 23, - }, - }, + Text: []byte(" "), }, { Type: TriviaTypeInlineComment, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 49, - Line: 2, - Column: 24, - }, - EndPos: ast.Position{ - Offset: 75, - Line: 2, - Column: 50, - }, - }, + Text: []byte("// test is in the same line"), }, }, }, @@ -951,34 +925,12 @@ func TestLexBasic(t *testing.T) { }, LeadingTrivia: []Trivia{ { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 76, - Line: 2, - Column: 51, - }, - EndPos: ast.Position{ - Offset: 76, - Line: 2, - Column: 51, - }, - }, + Type: TriviaTypeNewLine, + Text: []byte("\n"), }, { Type: TriviaTypeInlineComment, - Range: ast.Range{ - StartPos: ast.Position{ - Offset: 77, - Line: 3, - Column: 0, - }, - EndPos: ast.Position{ - Offset: 103, - Line: 3, - Column: 26, - }, - }, + Text: []byte("// test is in previous line"), }, }, TrailingTrivia: []Trivia{}, @@ -2070,6 +2022,19 @@ func TestLexString(t *testing.T) { }, Source: "\"", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -2077,15 +2042,6 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - }, }, }, }, @@ -2106,6 +2062,19 @@ func TestLexString(t *testing.T) { }, Source: "\"te", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -2113,15 +2082,6 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 4}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 4}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, - }, }, }, }, @@ -2196,6 +2156,19 @@ func TestLexString(t *testing.T) { }, Source: "\"\\", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -2203,15 +2176,6 @@ func TestLexString(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 3}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 3}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - }, - }, - }, }, }, }, @@ -2903,6 +2867,19 @@ func TestLexIntegerLiterals(t *testing.T) { }, Source: "0", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -2910,15 +2887,6 @@ func TestLexIntegerLiterals(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 0, Offset: 2}, EndPos: ast.Position{Line: 2, Column: 0, Offset: 2}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - }, }, }, }, @@ -3153,6 +3121,19 @@ func TestLexLineComment(t *testing.T) { testLex(t, ` foo // bar `, []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenIdentifier, @@ -3160,33 +3141,31 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, + }, + Source: "foo", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - { - Type: TriviaTypeInlineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, }, }, - Source: "foo", + Source: " ", + }, + { + Token: Token{ + Type: TokenLineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, + Source: "// bar ", }, { Token: Token{ @@ -3207,6 +3186,19 @@ func TestLexLineComment(t *testing.T) { t, " foo // bar \n baz", []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenIdentifier, @@ -3214,33 +3206,44 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, + }, + Source: "foo", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, - }, - }, - { - Type: TriviaTypeInlineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, + EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, }, }, - Source: "foo", + Source: " ", + }, + { + Token: Token{ + Type: TokenLineComment, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, + EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, + }, + }, + Source: "// bar ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: true, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, + EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, + }, + }, + Source: "\n ", }, { Token: Token{ @@ -3249,22 +3252,6 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 1, Offset: 14}, EndPos: ast.Position{Line: 2, Column: 3, Offset: 16}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - EndPos: ast.Position{Line: 1, Column: 12, Offset: 12}, - }, - }, - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 2, Column: 0, Offset: 13}, - EndPos: ast.Position{Line: 2, Column: 0, Offset: 13}, - }, - }, - }, }, Source: "baz", }, @@ -3297,14 +3284,19 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, + }, + tokenStream.Next(), + ) + + assert.Equal(t, + Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, }, tokenStream.Next(), @@ -3319,14 +3311,19 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, + }, + tokenStream.Next(), + ) + + assert.Equal(t, + Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, }, tokenStream.Next(), @@ -3353,7 +3350,6 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3365,7 +3361,6 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3383,14 +3378,19 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, + }, + tokenStream.Next(), + ) + + assert.Equal(t, + Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 3, Offset: 3}, + EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, }, tokenStream.Next(), @@ -3417,7 +3417,6 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3429,7 +3428,6 @@ func TestRevert(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, EndPos: ast.Position{Line: 1, Column: 5, Offset: 5}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3451,14 +3449,19 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, + }, + tokenStream.Next(), + ) + + assert.Equal(t, + Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, + EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, }, tokenStream.Next(), @@ -3488,7 +3491,6 @@ func TestEOFsAfterError(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, EndPos: ast.Position{Line: 1, Column: 2, Offset: 2}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3513,7 +3515,6 @@ func TestEOFsAfterEmptyInput(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, EndPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, - TrailingTrivia: []Trivia{}, }, tokenStream.Next(), ) @@ -3525,9 +3526,8 @@ func TestLimit(t *testing.T) { t.Parallel() var b strings.Builder - // TODO(preserve-comments): Token limit is affected, because comments are not treated as separate tokens for i := 0; i < 300000; i++ { - b.WriteString("x x ") + b.WriteString("x ") } code := b.String() From 2be52ed09696c1241301bf60fc400acfe894a898 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 13 Aug 2024 22:23:24 +0200 Subject: [PATCH 26/84] remove trivia structs --- runtime/old_parser/parser.go | 31 ------------ runtime/parser/lexer/lexer_test.go | 75 ------------------------------ runtime/parser/lexer/token.go | 13 +++--- runtime/parser/lexer/trivia.go | 19 -------- 4 files changed, 6 insertions(+), 132 deletions(-) delete mode 100644 runtime/parser/lexer/trivia.go diff --git a/runtime/old_parser/parser.go b/runtime/old_parser/parser.go index c0aed523c2..c55e5ca134 100644 --- a/runtime/old_parser/parser.go +++ b/runtime/old_parser/parser.go @@ -273,11 +273,6 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } -func (p *parser) triviaSource(trivia lexer.Trivia) []byte { - input := p.tokens.Input() - return trivia.Source(input) -} - func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } @@ -491,32 +486,6 @@ func (p *parser) endAmbiguity() { } } -func (p *parser) newCommentsFromTrivia(leadingTrivia, trailingTrivia []lexer.Trivia) ast.Comments { - comments := ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{}, - } - - for _, t := range leadingTrivia { - if isTriviaComment(t) { - comments.Leading = append(comments.Leading, ast.NewComment(p.memoryGauge, p.triviaSource(t))) - } - } - - for _, t := range trailingTrivia { - if isTriviaComment(t) { - comments.Trailing = append(comments.Trailing, ast.NewComment(p.memoryGauge, p.triviaSource(t))) - } - } - - return comments -} - -func isTriviaComment(t lexer.Trivia) bool { - // Other trivia types (space, newlines) are not needed in further stages of the compiler - return t.Type == lexer.TriviaTypeInlineComment || t.Type == lexer.TriviaTypeMultiLineComment -} - func ParseExpression( memoryGauge common.MemoryGauge, input []byte, diff --git a/runtime/parser/lexer/lexer_test.go b/runtime/parser/lexer/lexer_test.go index e5e1db5271..972585a45e 100644 --- a/runtime/parser/lexer/lexer_test.go +++ b/runtime/parser/lexer/lexer_test.go @@ -885,34 +885,6 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeInlineComment, - Text: []byte("// test is in next line "), - }, - { - Type: TriviaTypeNewLine, - Text: []byte("\n"), - }, - { - Type: TriviaTypeMultiLineComment, - Text: []byte("/* test is here */"), - }, - { - Type: TriviaTypeSpace, - Text: []byte(" "), - }, - }, - TrailingTrivia: []Trivia{ - { - Type: TriviaTypeSpace, - Text: []byte(" "), - }, - { - Type: TriviaTypeInlineComment, - Text: []byte("// test is in the same line"), - }, - }, }, Source: "test", }, @@ -923,17 +895,6 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeNewLine, - Text: []byte("\n"), - }, - { - Type: TriviaTypeInlineComment, - Text: []byte("// test is in previous line"), - }, - }, - TrailingTrivia: []Trivia{}, }, }, }, @@ -2226,22 +2187,6 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, EndPos: ast.Position{Line: 1, Column: 30, Offset: 30}, }, - LeadingTrivia: []Trivia{ - { - Type: TriviaTypeMultiLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - }, - { - Type: TriviaTypeSpace, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, - EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, - }, - }, - }, }, }, }, @@ -3157,16 +3102,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenEOF, @@ -3222,16 +3157,6 @@ func TestLexLineComment(t *testing.T) { }, Source: " ", }, - { - Token: Token{ - Type: TokenLineComment, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 5, Offset: 5}, - EndPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - }, - }, - Source: "// bar ", - }, { Token: Token{ Type: TokenSpace, diff --git a/runtime/parser/lexer/token.go b/runtime/parser/lexer/token.go index be31d69b6a..fca59a99f6 100644 --- a/runtime/parser/lexer/token.go +++ b/runtime/parser/lexer/token.go @@ -24,15 +24,14 @@ import ( type Token struct { SpaceOrError any + Type TokenType ast.Range - ast.Comments - Type TokenType - // LeadingTrivia up to and including the first contiguous sequence of newlines characters. - // Not tracked for space token, since those are usually ignored in the parser. - LeadingTrivia []Trivia - // TrailingTrivia up to, but not including, the next newline character. + // TODO(preserve-comments): This is currently not true (first comment), + // as leading comments are just all comments after the trailing comments of the previous token. + // Leading comments span up to and including the first contiguous sequence of newlines characters. + // Trailing comments span up to, but not including, the next newline character. // Not tracked for space token, since those are usually ignored in the parser. - TrailingTrivia []Trivia + ast.Comments } func (t Token) Is(ty TokenType) bool { diff --git a/runtime/parser/lexer/trivia.go b/runtime/parser/lexer/trivia.go deleted file mode 100644 index 79f00a4c39..0000000000 --- a/runtime/parser/lexer/trivia.go +++ /dev/null @@ -1,19 +0,0 @@ -package lexer - -import "github.com/onflow/cadence/runtime/ast" - -type Trivia struct { - Type TriviaType - // Position within the source code (includes opening/closing comment characters in case of comment trivia type) - ast.Range -} - -type TriviaType uint8 - -// TODO(preserve-comments): Refactor to Comment, since we now still track spaces using tokens -const ( - TriviaTypeUnknown TriviaType = iota - TriviaTypeInlineComment - TriviaTypeMultiLineComment - TriviaTypeSpace -) From 1c287071f7c9187fb495ce93b9bfe1c19ae5297f Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 16 Aug 2024 20:13:23 +0200 Subject: [PATCH 27/84] include comments in event declaration --- runtime/ast/block.go | 9 +-- runtime/ast/comments.go | 7 +++ runtime/ast/composite.go | 3 + runtime/ast/function_declaration_test.go | 5 +- runtime/interpreter/interpreter_statement.go | 1 + runtime/old_parser/declaration.go | 3 + runtime/old_parser/statement.go | 3 + runtime/parser/declaration.go | 11 +++- runtime/parser/declaration_test.go | 59 +++++++++++++++++++- runtime/parser/function.go | 29 ++++++---- runtime/parser/parser.go | 5 -- runtime/parser/parser_test.go | 33 ++++++++++- runtime/parser/statement.go | 7 ++- runtime/parser/transaction.go | 3 +- runtime/sema/check_switch.go | 1 + 15 files changed, 146 insertions(+), 33 deletions(-) diff --git a/runtime/ast/block.go b/runtime/ast/block.go index 69518f8077..9528fbfcc5 100644 --- a/runtime/ast/block.go +++ b/runtime/ast/block.go @@ -34,14 +34,7 @@ type Block struct { var _ Element = &Block{} -// TODO(preserve-comments): Migrate and remove -func NewBlockWithComments(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { - block := NewBlock(memoryGauge, statements, astRange) - block.Comments = comments - return block -} - -func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range) *Block { +func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) return &Block{ diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index f7d94d6540..c46743256c 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -10,6 +10,13 @@ type Comments struct { Trailing []*Comment `json:"-"` } +func (c Comments) PackToList() []*Comment { + var comments []*Comment + comments = append(comments, c.Leading...) + comments = append(comments, c.Trailing...) + return comments +} + type Comment struct { source []byte } diff --git a/runtime/ast/composite.go b/runtime/ast/composite.go index dae942b6af..d6e74fec43 100644 --- a/runtime/ast/composite.go +++ b/runtime/ast/composite.go @@ -57,6 +57,7 @@ type CompositeDeclaration struct { Range Access Access CompositeKind common.CompositeKind + Comments } var _ Element = &CompositeDeclaration{} @@ -73,6 +74,7 @@ func NewCompositeDeclaration( members *Members, docString string, declarationRange Range, + comments Comments, ) *CompositeDeclaration { common.UseMemory(memoryGauge, common.CompositeDeclarationMemoryUsage) @@ -84,6 +86,7 @@ func NewCompositeDeclaration( Members: members, DocString: docString, Range: declarationRange, + Comments: comments, } } diff --git a/runtime/ast/function_declaration_test.go b/runtime/ast/function_declaration_test.go index 521df5bf8b..6531a8aab1 100644 --- a/runtime/ast/function_declaration_test.go +++ b/runtime/ast/function_declaration_test.go @@ -114,7 +114,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, Comments: Comments{ - Leading: []Comment{NewComment(nil, []byte("///test"))}, + Leading: []*Comment{NewComment(nil, []byte("///test"))}, }, StartPos: Position{Offset: 34, Line: 35, Column: 36}, } @@ -587,7 +587,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { }, }, Comments: Comments{ - Leading: []Comment{NewComment(nil, []byte("///test"))}, + Leading: []*Comment{NewComment(nil, []byte("///test"))}, }, StartPos: Position{Offset: 34, Line: 35, Column: 36}, }, @@ -596,6 +596,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { actual, err := json.Marshal(decl) require.NoError(t, err) + // TODO(preserve-comments): Do we need to include comments in the JSON AST? assert.JSONEq(t, // language=json ` diff --git a/runtime/interpreter/interpreter_statement.go b/runtime/interpreter/interpreter_statement.go index 9170eaa0ce..c5cc588435 100644 --- a/runtime/interpreter/interpreter_statement.go +++ b/runtime/interpreter/interpreter_statement.go @@ -192,6 +192,7 @@ func (interpreter *Interpreter) VisitSwitchStatement(switchStatement *ast.Switch interpreter, switchCase.Statements, ast.EmptyRange, + ast.Comments{}, ) result := interpreter.visitBlock(block) diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index 56747a48f9..f3bd02c43a 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -100,6 +100,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { accessPos, staticPos, nativePos, + "", ) case keywordImport: @@ -824,6 +825,7 @@ func parseEventDeclaration( startPos, parameterList.EndPos, ), + ast.Comments{}, ), nil } @@ -1060,6 +1062,7 @@ func parseCompositeOrInterfaceDeclaration( members, docString, declarationRange, + ast.Comments{}, ), nil } } diff --git a/runtime/old_parser/statement.go b/runtime/old_parser/statement.go index dfa73bb617..c0c3bfc887 100644 --- a/runtime/old_parser/statement.go +++ b/runtime/old_parser/statement.go @@ -359,6 +359,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.Comments{}, ) result = outer } @@ -476,6 +477,7 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{}, ), nil } @@ -535,6 +537,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{}, ), preConditions, postConditions, diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 2afa1c127d..d5eabf9422 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -925,8 +925,8 @@ func parseEventDeclaration( accessPos *ast.Position, docString string, ) (*ast.CompositeDeclaration, error) { - - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -943,7 +943,7 @@ func parseEventDeclaration( // if this is a `ResourceDestroyed` event (i.e., a default event declaration), parse default arguments parseDefaultArguments := ast.IsResourceDestructionDefaultEvent(identifier.Identifier) - parameterList, err := parseParameterList(p, parseDefaultArguments) + parameterList, endComments, err := parseParameterList(p, parseDefaultArguments) if err != nil { return nil, err } @@ -987,6 +987,10 @@ func parseEventDeclaration( startPos, parameterList.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.Leading, + Trailing: endComments.PackToList(), + }, ), nil } @@ -1408,6 +1412,7 @@ func parseCompositeOrInterfaceDeclaration( members, docString, declarationRange, + ast.Comments{}, ), nil } } diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index ada66107ea..7f990bb297 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -389,7 +389,9 @@ func TestParseParameterList(t *testing.T) { nil, []byte(input), func(p *parser) (*ast.ParameterList, error) { - return parseParameterList(p, false) + // TODO(preserve-comments): Do we care about these comments (second return value)? + parameters, _, err := parseParameterList(p, false) + return parameters, err }, Config{}, ) @@ -2484,6 +2486,61 @@ func TestParseEvent(t *testing.T) { ) }) + t.Run("no parameters, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before E +event E() // After E +// Should be ignored +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Offset: 19, Line: 3, Column: 6}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After E")), + }, + }, + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + Kind: common.DeclarationKindInitializer, + FunctionDeclaration: &ast.FunctionDeclaration{ + Access: ast.AccessNotSpecified, + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + }, + StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, + }, + }, + }, + ), + Range: ast.Range{ + StartPos: ast.Position{Offset: 13, Line: 3, Column: 0}, + EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, + }, + }, + }, + result, + ) + }) + t.Run("two parameters, private", func(t *testing.T) { t.Parallel() diff --git a/runtime/parser/function.go b/runtime/parser/function.go index 1667aa3a25..051e1ff66a 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -32,23 +32,27 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { return ast.FunctionPurityUnspecified } -func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { +func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, ast.Comments, error) { var parameters []*ast.Parameter + var comments ast.Comments p.skipSpace() if !p.current.Is(lexer.TokenParenOpen) { - return nil, p.syntaxError( + return nil, comments, p.syntaxError( "expected %s as start of parameter list, got %s", lexer.TokenParenOpen, p.current.Type, ) } - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos // Skip the opening paren p.next() + comments.Leading = startToken.PackToList() + var endPos ast.Position expectParameter := true @@ -65,7 +69,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } parameter, err := parseParameter(p, expectDefaultArguments) if err != nil { - return nil, err + return nil, comments, err } parameters = append(parameters, parameter) @@ -73,7 +77,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL case lexer.TokenComma: if expectParameter { - return nil, p.syntaxError( + return nil, comments, p.syntaxError( "expected parameter or end of parameter list, got %s", p.current.Type, ) @@ -83,25 +87,27 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL expectParameter = true case lexer.TokenParenClose: - endPos = p.current.EndPos + endToken := p.current + endPos = endToken.EndPos // Skip the closing paren p.next() + comments.Trailing = endToken.PackToList() atEnd = true case lexer.TokenEOF: - return nil, p.syntaxError( + return nil, comments, p.syntaxError( "missing %s at end of parameter list", lexer.TokenParenClose, ) default: if expectParameter { - return nil, p.syntaxError( + return nil, comments, p.syntaxError( "expected parameter or end of parameter list, got %s", p.current.Type, ) } else { - return nil, p.syntaxError( + return nil, comments, p.syntaxError( "expected comma or end of parameter list, got %s", p.current.Type, ) @@ -117,7 +123,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL startPos, endPos, ), - ), nil + ), comments, nil } func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, error) { @@ -388,7 +394,8 @@ func parseFunctionParameterListAndRest( ) { // Parameter list - parameterList, err = parseParameterList(p, false) + // TODO(preserve-comments): Do we care about these comments (second return value)? + parameterList, _, err = parseParameterList(p, false) if err != nil { return } diff --git a/runtime/parser/parser.go b/runtime/parser/parser.go index 6b9bf89e8b..5b82ab31be 100644 --- a/runtime/parser/parser.go +++ b/runtime/parser/parser.go @@ -283,11 +283,6 @@ func (p *parser) tokenSource(token lexer.Token) []byte { return token.Source(input) } -func (p *parser) triviaSource(trivia lexer.Trivia) []byte { - input := p.tokens.Input() - return trivia.Source(input) -} - func (p *parser) currentTokenSource() []byte { return p.tokenSource(p.current) } diff --git a/runtime/parser/parser_test.go b/runtime/parser/parser_test.go index e9ccecd5b9..e3626c525e 100644 --- a/runtime/parser/parser_test.go +++ b/runtime/parser/parser_test.go @@ -1014,10 +1014,41 @@ func TestParseWhitespaceAtEnd(t *testing.T) { assert.Empty(t, errs) } -func TestParseTrivia(t *testing.T) { +func TestParseComments(t *testing.T) { t.Parallel() + t.Run("special function declaration", func(t *testing.T) { + res, errs := ParseDeclarations( + nil, + []byte(` +/// Before MyEvent +event MyEvent() // After MyEvent +/// Before prepare +prepare() {} // After prepare +/// Before pre +pre {} // After pre +/// Before execute +execute {} // After execute +/// Before post +post {} // After post +`), + Config{}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + event, ok := res[0].(*ast.SpecialFunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, + Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, + }, event.FunctionDeclaration.Comments) + assert.Equal(t, " Before MyEvent", event.FunctionDeclaration.DeclarationDocString()) + + }) + t.Run("function declaration", func(t *testing.T) { res, errs := ParseProgram( nil, diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index efb3542012..7a66d4a3e4 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -388,6 +388,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), + ast.Comments{}, ) result = outer } @@ -505,6 +506,10 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: endToken.Comments.PackToList(), + }, ), nil } @@ -556,7 +561,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { return ast.NewFunctionBlock( p.memoryGauge, - ast.NewBlockWithComments( + ast.NewBlock( p.memoryGauge, statements, ast.NewRange( diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index d5ebfd23b6..4f28d05fcb 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -53,7 +53,8 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var err error if p.current.Is(lexer.TokenParenOpen) { - parameterList, err = parseParameterList(p, false) + // TODO(preserve-comments): Do we care about these comments (second return value)? + parameterList, _, err = parseParameterList(p, false) if err != nil { return nil, err } diff --git a/runtime/sema/check_switch.go b/runtime/sema/check_switch.go index 84d6670f25..054c02264c 100644 --- a/runtime/sema/check_switch.go +++ b/runtime/sema/check_switch.go @@ -185,6 +185,7 @@ func (checker *Checker) checkSwitchCaseStatements(switchCase *ast.SwitchCase) { switchCase.Statements[0].StartPosition(), switchCase.EndPos, ), + ast.Comments{}, ) checker.checkBlock(block) } From 1f1c78fcc4ec98ee7460ece8b1a308656e994f6e Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 21 Sep 2024 14:07:12 +0200 Subject: [PATCH 28/84] update variable declaration parsing --- runtime/ast/comments.go | 15 ++++++ runtime/ast/expression.go | 3 ++ runtime/ast/function_declaration.go | 13 +---- runtime/ast/variable_declaration.go | 8 ++-- runtime/old_parser/declaration.go | 2 +- runtime/old_parser/expression.go | 2 +- runtime/parser/declaration.go | 17 +++++-- runtime/parser/declaration_test.go | 55 +++++++++++++++++++++- runtime/parser/expression.go | 16 +++---- runtime/parser/statement.go | 2 +- runtime/sema/check_variable_declaration.go | 2 +- runtime/sema/positioninfo.go | 2 +- tools/maprange/go.sum | 5 ++ 13 files changed, 108 insertions(+), 34 deletions(-) diff --git a/runtime/ast/comments.go b/runtime/ast/comments.go index c46743256c..2daa66cde5 100644 --- a/runtime/ast/comments.go +++ b/runtime/ast/comments.go @@ -3,6 +3,7 @@ package ast import ( "bytes" "github.com/onflow/cadence/runtime/common" + "strings" ) type Comments struct { @@ -17,6 +18,20 @@ func (c Comments) PackToList() []*Comment { return comments } +// LeadingDocString prints the leading doc comments to string +func (c Comments) LeadingDocString() string { + var s strings.Builder + for _, comment := range c.Leading { + if comment.Doc() { + if s.Len() > 0 { + s.WriteRune('\n') + } + s.Write(comment.Text()) + } + } + return s.String() +} + type Comment struct { source []byte } diff --git a/runtime/ast/expression.go b/runtime/ast/expression.go index 0142b92b20..992adf37e2 100644 --- a/runtime/ast/expression.go +++ b/runtime/ast/expression.go @@ -227,6 +227,7 @@ type IntegerExpression struct { PositiveLiteral []byte Range Base int + Comments } var _ Element = &IntegerExpression{} @@ -238,6 +239,7 @@ func NewIntegerExpression( value *big.Int, base int, tokenRange Range, + comments Comments, ) *IntegerExpression { common.UseMemory(gauge, common.IntegerExpressionMemoryUsage) @@ -246,6 +248,7 @@ func NewIntegerExpression( Value: value, Base: base, Range: tokenRange, + Comments: comments, } } diff --git a/runtime/ast/function_declaration.go b/runtime/ast/function_declaration.go index e9d384c50d..d5598077a0 100644 --- a/runtime/ast/function_declaration.go +++ b/runtime/ast/function_declaration.go @@ -20,8 +20,6 @@ package ast import ( "encoding/json" - "strings" - "github.com/turbolent/prettier" "github.com/onflow/cadence/runtime/common" @@ -209,16 +207,7 @@ func (d *FunctionDeclaration) DeclarationMembers() *Members { } func (d *FunctionDeclaration) DeclarationDocString() string { - var s strings.Builder - for _, comment := range d.Comments.Leading { - if comment.Doc() { - if s.Len() > 0 { - s.WriteRune('\n') - } - s.Write(comment.Text()) - } - } - return s.String() + return d.Comments.LeadingDocString() } func (d *FunctionDeclaration) Doc() prettier.Doc { diff --git a/runtime/ast/variable_declaration.go b/runtime/ast/variable_declaration.go index bc82b55a5a..43620b0289 100644 --- a/runtime/ast/variable_declaration.go +++ b/runtime/ast/variable_declaration.go @@ -33,11 +33,11 @@ type VariableDeclaration struct { Transfer *Transfer SecondTransfer *Transfer ParentIfStatement *IfStatement `json:"-"` - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access IsConstant bool + Comments } var _ Element = &VariableDeclaration{} @@ -55,7 +55,7 @@ func NewVariableDeclaration( startPos Position, secondTransfer *Transfer, secondValue Expression, - docString string, + comments Comments, ) *VariableDeclaration { common.UseMemory(gauge, common.VariableDeclarationMemoryUsage) @@ -69,7 +69,7 @@ func NewVariableDeclaration( StartPos: startPos, SecondTransfer: secondTransfer, SecondValue: secondValue, - DocString: docString, + Comments: comments, } } @@ -127,7 +127,7 @@ func (d *VariableDeclaration) DeclarationMembers() *Members { } func (d *VariableDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } var varKeywordDoc prettier.Doc = prettier.Text("var") diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index f3bd02c43a..3e22a447ac 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -415,7 +415,7 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - docString, + ast.Comments{}, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) diff --git a/runtime/old_parser/expression.go b/runtime/old_parser/expression.go index b4fd4c2a12..7a2f8638fb 100644 --- a/runtime/old_parser/expression.go +++ b/runtime/old_parser/expression.go @@ -1771,7 +1771,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange, ast.Comments{}) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index d5eabf9422..8fa9019941 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -102,7 +102,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") } - return parseVariableDeclaration(p, access, accessPos, docString) + return parseVariableDeclaration(p, access, accessPos) case KeywordFun: return parseFunctionDeclaration( @@ -512,10 +512,10 @@ func parseVariableDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, ) (*ast.VariableDeclaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -525,6 +525,8 @@ func parseVariableDeclaration( // Skip the `let` or `var` keyword p.nextSemanticToken() + identifierToken := p.current + identifier, err := p.nonReservedIdentifier("after start of variable declaration") if err != nil { return nil, err @@ -546,6 +548,8 @@ func parseVariableDeclaration( } p.skipSpace() + + transferToken := p.current transfer := parseTransfer(p) if transfer == nil { return nil, p.syntaxError("expected transfer") @@ -578,7 +582,12 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - docString, + ast.Comments{ + Leading: append( + append(startToken.PackToList(), identifierToken.PackToList()...), + transferToken.PackToList()..., + ), + }, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 7f990bb297..84bde4008b 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -72,6 +72,59 @@ func TestParseVariableDeclaration(t *testing.T) { ) }) + t.Run("var, no type annotation, copy, one value, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before x +var x = /* Before 1 */ 1 // After 1 +// Ignored +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.VariableDeclaration{ + Access: ast.AccessNotSpecified, + IsConstant: false, + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 4, Offset: 17}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before x")), + ast.NewComment(nil, []byte("/* Before 1 */")), + }, + Trailing: []*ast.Comment{}, + }, + Value: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + EndPos: ast.Position{Line: 3, Column: 23, Offset: 36}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After 1")), + }, + }, + }, + Transfer: &ast.Transfer{ + Operation: ast.TransferOperationCopy, + Pos: ast.Position{Line: 3, Column: 6, Offset: 19}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + }, + }, + result, + ) + }) + t.Run("var, no type annotation, copy, one value, access(all)", func(t *testing.T) { t.Parallel() @@ -2493,7 +2546,7 @@ func TestParseEvent(t *testing.T) { result, errs := testParseDeclarations(` // Before E event E() // After E -// Should be ignored +// Ignored `) require.Empty(t, errs) diff --git a/runtime/parser/expression.go b/runtime/parser/expression.go index 87db3ed912..8834d05e11 100644 --- a/runtime/parser/expression.go +++ b/runtime/parser/expression.go @@ -360,7 +360,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindBinary, - token.Range, + token, ), nil }, }) @@ -374,7 +374,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindOctal, - token.Range, + token, ), nil }, }) @@ -388,7 +388,7 @@ func init() { literal, literal, common.IntegerLiteralKindDecimal, - token.Range, + token, ), nil }, }) @@ -402,7 +402,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindHexadecimal, - token.Range, + token, ), nil }, }) @@ -416,7 +416,7 @@ func init() { literal, literal[2:], common.IntegerLiteralKindUnknown, - token.Range, + token, ), nil }, }) @@ -1730,7 +1730,7 @@ func parseHex(r rune) rune { return -1 } -func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, tokenRange ast.Range) *ast.IntegerExpression { +func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLiteralKind, token lexer.Token) *ast.IntegerExpression { report := func(invalidKind InvalidNumberLiteralKind) { p.report( @@ -1739,7 +1739,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit InvalidIntegerLiteralKind: invalidKind, // NOTE: not using text, because it has the base-prefix stripped Literal: string(literal), - Range: tokenRange, + Range: token.Range, }, ) } @@ -1789,7 +1789,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, token.Range, token.Comments) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index 7a66d4a3e4..6697f94905 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -311,7 +311,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil, "") + parseVariableDeclaration(p, ast.AccessNotSpecified, nil) if err != nil { return nil, err } diff --git a/runtime/sema/check_variable_declaration.go b/runtime/sema/check_variable_declaration.go index b451416262..fb26450aa6 100644 --- a/runtime/sema/check_variable_declaration.go +++ b/runtime/sema/check_variable_declaration.go @@ -225,7 +225,7 @@ func (checker *Checker) declareVariableDeclaration(declaration *ast.VariableDecl variable, err := checker.valueActivations.declare(variableDeclaration{ identifier: identifier, ty: declarationType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), access: checker.accessFromAstAccess(declaration.Access), kind: declaration.DeclarationKind(), pos: declaration.Identifier.Pos, diff --git a/runtime/sema/positioninfo.go b/runtime/sema/positioninfo.go index 9b0a8a5687..53f4fb0c0e 100644 --- a/runtime/sema/positioninfo.go +++ b/runtime/sema/positioninfo.go @@ -246,7 +246,7 @@ func (i *PositionInfo) recordVariableDeclarationRange( Identifier: identifier, DeclarationKind: declaration.DeclarationKind(), Type: declarationType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), }, ) } diff --git a/tools/maprange/go.sum b/tools/maprange/go.sum index 05680b3128..7a9273fb41 100644 --- a/tools/maprange/go.sum +++ b/tools/maprange/go.sum @@ -1,6 +1,11 @@ +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA= golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= golang.org/x/tools v0.21.0 h1:qc0xYgIbsSDt9EyWz05J5wfa7LOVW0YTLOXrqdLAWIw= golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= From 79ecacf6be64a34c590e821e2ebfb9d46da4a60c Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 25 Sep 2024 12:22:25 +0200 Subject: [PATCH 29/84] update parameter parsing --- runtime/ast/parameter.go | 3 + runtime/ast/parameterlist.go | 3 + runtime/ast/type.go | 16 ++++ runtime/old_parser/function.go | 2 + runtime/parser/declaration.go | 5 +- runtime/parser/declaration_test.go | 144 ++++++++++++++++++++++++++++- runtime/parser/function.go | 70 +++++++++----- runtime/parser/transaction.go | 3 +- runtime/parser/type.go | 30 +++++- 9 files changed, 240 insertions(+), 36 deletions(-) diff --git a/runtime/ast/parameter.go b/runtime/ast/parameter.go index 937ae2a3aa..34c1e6f68b 100644 --- a/runtime/ast/parameter.go +++ b/runtime/ast/parameter.go @@ -32,6 +32,7 @@ type Parameter struct { Label string Identifier Identifier StartPos Position `json:"-"` + Comments } func NewParameter( @@ -41,6 +42,7 @@ func NewParameter( typeAnnotation *TypeAnnotation, defaultArgument Expression, startPos Position, + comments Comments, ) *Parameter { common.UseMemory(gauge, common.ParameterMemoryUsage) return &Parameter{ @@ -49,6 +51,7 @@ func NewParameter( TypeAnnotation: typeAnnotation, DefaultArgument: defaultArgument, StartPos: startPos, + Comments: comments, } } diff --git a/runtime/ast/parameterlist.go b/runtime/ast/parameterlist.go index 97eef6a0f0..638f0aee18 100644 --- a/runtime/ast/parameterlist.go +++ b/runtime/ast/parameterlist.go @@ -30,6 +30,7 @@ type ParameterList struct { _parametersByIdentifier map[string]*Parameter Parameters []*Parameter Range + Comments once sync.Once } @@ -37,11 +38,13 @@ func NewParameterList( gauge common.MemoryGauge, parameters []*Parameter, astRange Range, + comments Comments, ) *ParameterList { common.UseMemory(gauge, common.ParameterListMemoryUsage) return &ParameterList{ Parameters: parameters, Range: astRange, + Comments: comments, } } diff --git a/runtime/ast/type.go b/runtime/ast/type.go index 095653afd4..d893b50f77 100644 --- a/runtime/ast/type.go +++ b/runtime/ast/type.go @@ -109,6 +109,7 @@ func IsEmptyType(t Type) bool { type NominalType struct { NestedIdentifiers []Identifier `json:",omitempty"` Identifier Identifier + Comments } var _ Type = &NominalType{} @@ -125,6 +126,21 @@ func NewNominalType( } } +// TODO(preserve-comments): Should we remove this and use only NewNominalType (requires updating all dependants) +func NewNominalTypeWithComments( + memoryGauge common.MemoryGauge, + identifier Identifier, + nestedIdentifiers []Identifier, + comments Comments, +) *NominalType { + common.UseMemory(memoryGauge, common.NominalTypeMemoryUsage) + return &NominalType{ + Identifier: identifier, + NestedIdentifiers: nestedIdentifiers, + Comments: comments, + } +} + func (*NominalType) isType() {} func (t *NominalType) String() string { diff --git a/runtime/old_parser/function.go b/runtime/old_parser/function.go index 5c110f9273..216ff0bb25 100644 --- a/runtime/old_parser/function.go +++ b/runtime/old_parser/function.go @@ -108,6 +108,7 @@ func parseParameterList(p *parser) (*ast.ParameterList, error) { startPos, endPos, ), + ast.Comments{}, ), nil } @@ -167,6 +168,7 @@ func parseParameter(p *parser) (*ast.Parameter, error) { typeAnnotation, nil, startPos, + ast.Comments{}, ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 8fa9019941..8a17964e7d 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -952,7 +952,7 @@ func parseEventDeclaration( // if this is a `ResourceDestroyed` event (i.e., a default event declaration), parse default arguments parseDefaultArguments := ast.IsResourceDestructionDefaultEvent(identifier.Identifier) - parameterList, endComments, err := parseParameterList(p, parseDefaultArguments) + parameterList, err := parseParameterList(p, parseDefaultArguments) if err != nil { return nil, err } @@ -997,8 +997,7 @@ func parseEventDeclaration( parameterList.EndPos, ), ast.Comments{ - Leading: startToken.Comments.Leading, - Trailing: endComments.PackToList(), + Leading: startToken.Comments.Leading, }, ), nil } diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 84bde4008b..9c6d3ea19a 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -442,8 +442,7 @@ func TestParseParameterList(t *testing.T) { nil, []byte(input), func(p *parser) (*ast.ParameterList, error) { - // TODO(preserve-comments): Do we care about these comments (second return value)? - parameters, _, err := parseParameterList(p, false) + parameters, err := parseParameterList(p, false) return parameters, err }, Config{}, @@ -524,6 +523,62 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("one, resource type, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := parse(`( a : /* After colon */ +// Before type +@Int /* After type */ )`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{Line: 1, Column: 2, Offset: 2}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: true, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 1, Offset: 41}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + // This comment should be attached to Type instead of TypeAnnotation + // (even tho it's technically attached to the @ resource symbol) for simplicity. + ast.NewComment(nil, []byte("// Before type")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After type */")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 40}, + }, + StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After colon */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 3, Column: 22, Offset: 62}, + }, + }, + result, + ) + }) + t.Run("one, with argument label", func(t *testing.T) { t.Parallel() @@ -618,6 +673,91 @@ func TestParseParameterList(t *testing.T) { ) }) + t.Run("two, with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := parse(`( /* Before param b */ a b : /* Before b type annotation */ Int /* After param b */, +// Before param c +c /* After c identifier */ : Int // After param c +)`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + Label: "a", + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Line: 1, Column: 25, Offset: 25}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After param b */")), + }, + }, + }, + StartPos: ast.Position{Line: 1, Column: 60, Offset: 60}, + }, + StartPos: ast.Position{Line: 1, Column: 23, Offset: 23}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* Before param b */")), + ast.NewComment(nil, []byte("/* Before b type annotation */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + { + Label: "", + Identifier: ast.Identifier{ + Identifier: "c", + Pos: ast.Position{Line: 3, Column: 0, Offset: 104}, + }, + TypeAnnotation: &ast.TypeAnnotation{ + IsResource: false, + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After param c ")), + }, + }, + }, + StartPos: ast.Position{Line: 3, Column: 29, Offset: 133}, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 104}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before param c")), + ast.NewComment(nil, []byte("/* After c identifier */")), + }, + Trailing: []*ast.Comment{}, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, + EndPos: ast.Position{Line: 4, Column: 0, Offset: 155}, + }, + }, + result, + ) + }) + t.Run("two, with and without argument label, missing comma", func(t *testing.T) { t.Parallel() diff --git a/runtime/parser/function.go b/runtime/parser/function.go index 051e1ff66a..e10325d152 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -32,14 +32,15 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { return ast.FunctionPurityUnspecified } -func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, ast.Comments, error) { +func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { var parameters []*ast.Parameter + var endToken lexer.Token var comments ast.Comments p.skipSpace() if !p.current.Is(lexer.TokenParenOpen) { - return nil, comments, p.syntaxError( + return nil, p.syntaxError( "expected %s as start of parameter list, got %s", lexer.TokenParenOpen, p.current.Type, @@ -47,14 +48,9 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } startToken := p.current - startPos := startToken.StartPos // Skip the opening paren p.next() - comments.Leading = startToken.PackToList() - - var endPos ast.Position - expectParameter := true atEnd := false @@ -69,7 +65,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } parameter, err := parseParameter(p, expectDefaultArguments) if err != nil { - return nil, comments, err + return nil, err } parameters = append(parameters, parameter) @@ -77,7 +73,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL case lexer.TokenComma: if expectParameter { - return nil, comments, p.syntaxError( + return nil, p.syntaxError( "expected parameter or end of parameter list, got %s", p.current.Type, ) @@ -87,27 +83,25 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL expectParameter = true case lexer.TokenParenClose: - endToken := p.current - endPos = endToken.EndPos + endToken = p.current // Skip the closing paren p.next() - comments.Trailing = endToken.PackToList() atEnd = true case lexer.TokenEOF: - return nil, comments, p.syntaxError( + return nil, p.syntaxError( "missing %s at end of parameter list", lexer.TokenParenClose, ) default: if expectParameter { - return nil, comments, p.syntaxError( + return nil, p.syntaxError( "expected parameter or end of parameter list, got %s", p.current.Type, ) } else { - return nil, comments, p.syntaxError( + return nil, p.syntaxError( "expected comma or end of parameter list, got %s", p.current.Type, ) @@ -115,21 +109,43 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } } + if len(parameters) == 0 { + comments.Leading = append( + comments.Leading, + startToken.Comments.PackToList()..., + ) + } else { + comments.Leading = append( + comments.Leading, + startToken.Comments.Leading..., + ) + + var patched []*ast.Comment + patched = append(patched, startToken.Comments.Trailing...) + patched = append(patched, parameters[0].Comments.Leading...) + parameters[0].Comments.Leading = patched + } + comments.Trailing = append( + comments.Trailing, + endToken.Comments.PackToList()..., + ) + return ast.NewParameterList( p.memoryGauge, parameters, ast.NewRange( p.memoryGauge, - startPos, - endPos, + startToken.StartPos, + endToken.EndPos, ), - ), comments, nil + comments, + ), nil } func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, error) { p.skipSpace() - startPos := p.current.StartPos + startToken := p.current argumentLabel := "" identifier, err := p.nonReservedIdentifier("for argument label or parameter name") @@ -156,11 +172,12 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro p.nextSemanticToken() } - if !p.current.Is(lexer.TokenColon) { + colonToken := p.current + if !colonToken.Is(lexer.TokenColon) { return nil, p.syntaxError( "expected %s after parameter name, got %s", lexer.TokenColon, - p.current.Type, + colonToken.Type, ) } @@ -203,7 +220,13 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro identifier, typeAnnotation, defaultArgument, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: append( + startToken.Comments.PackToList(), + colonToken.Comments.PackToList()..., + ), + }, ), nil } @@ -394,8 +417,7 @@ func parseFunctionParameterListAndRest( ) { // Parameter list - // TODO(preserve-comments): Do we care about these comments (second return value)? - parameterList, _, err = parseParameterList(p, false) + parameterList, err = parseParameterList(p, false) if err != nil { return } diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index 4f28d05fcb..d5ebfd23b6 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -53,8 +53,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD var err error if p.current.Is(lexer.TokenParenOpen) { - // TODO(preserve-comments): Do we care about these comments (second return value)? - parameterList, _, err = parseParameterList(p, false) + parameterList, err = parseParameterList(p, false) if err != nil { return nil, err } diff --git a/runtime/parser/type.go b/runtime/parser/type.go index d44467c7aa..ce8ee852fe 100644 --- a/runtime/parser/type.go +++ b/runtime/parser/type.go @@ -168,12 +168,14 @@ func defineParenthesizedTypes() { func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, error) { var nestedIdentifiers []ast.Identifier + // Stores the last token at the end of the loop + var nestedToken *lexer.Token for p.current.Is(lexer.TokenDot) { // Skip the dot p.next() - nestedToken := p.current + nestedToken = &p.current if !nestedToken.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( @@ -183,7 +185,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, ) } - nestedIdentifier := p.tokenToIdentifier(nestedToken) + nestedIdentifier := p.tokenToIdentifier(*nestedToken) // Skip the identifier p.next() @@ -195,10 +197,21 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, } - return ast.NewNominalType( + var trailingComments []*ast.Comment + if nestedToken == nil { + trailingComments = token.Comments.Trailing + } else { + trailingComments = nestedToken.Comments.PackToList() + } + + return ast.NewNominalTypeWithComments( p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, + ast.Comments{ + Leading: token.Comments.Leading, + Trailing: trailingComments, + }, ), nil } @@ -731,13 +744,20 @@ func defaultTypeMetaLeftDenotation( func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { p.skipSpace() - startPos := p.current.StartPos + startToken := p.current isResource := false if p.current.Is(lexer.TokenAt) { // Skip the `@` p.next() isResource = true + + // Append @ token comments to type token, + // so that we don't need to store them in TypeAnnotation node. + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startToken.Comments.Leading...) + leadingComments = append(leadingComments, p.current.Comments.Leading...) + p.current.Comments.Leading = leadingComments } ty, err := parseType(p, lowestBindingPower) @@ -749,7 +769,7 @@ func parseTypeAnnotation(p *parser) (*ast.TypeAnnotation, error) { p.memoryGauge, isResource, ty, - startPos, + startToken.StartPos, ), nil } From 63155edf687e2bc575e982f0306a6e7a76e4ee69 Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 27 Sep 2024 11:44:54 +0200 Subject: [PATCH 30/84] update variable declaration parsing --- runtime/ast/import.go | 3 ++ runtime/old_parser/declaration.go | 1 + runtime/parser/declaration.go | 24 +++++++++----- runtime/parser/declaration_test.go | 52 +++++++++++++++++++++++++++--- 4 files changed, 68 insertions(+), 12 deletions(-) diff --git a/runtime/ast/import.go b/runtime/ast/import.go index 312cc9903f..17f4f8da48 100644 --- a/runtime/ast/import.go +++ b/runtime/ast/import.go @@ -33,6 +33,7 @@ type ImportDeclaration struct { Identifiers []Identifier Range LocationPos Position + Comments } var _ Element = &ImportDeclaration{} @@ -44,6 +45,7 @@ func NewImportDeclaration( location common.Location, declRange Range, locationPos Position, + comments Comments, ) *ImportDeclaration { common.UseMemory(gauge, common.ImportDeclarationMemoryUsage) @@ -52,6 +54,7 @@ func NewImportDeclaration( Location: location, Range: declRange, LocationPos: locationPos, + Comments: comments, } } diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index 3e22a447ac..80276688c9 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -698,6 +698,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { endPos, ), locationPos, + ast.Comments{}, ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 8a17964e7d..6285b25a11 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -658,17 +658,19 @@ func parsePragmaDeclaration(p *parser) (*ast.PragmaDeclaration, error) { // ( string | hexadecimalLiteral | identifier ) func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { - startPosition := p.current.StartPos + startToken := p.current var identifiers []ast.Identifier var location common.Location var locationPos ast.Position var endPos ast.Position + var trailingComments []*ast.Comment parseStringOrAddressLocation := func() { locationPos = p.current.StartPos endPos = p.current.EndPos + trailingComments = p.current.Comments.Trailing switch p.current.Type { case lexer.TokenString: @@ -687,10 +689,11 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { p.next() } - setIdentifierLocation := func(identifier ast.Identifier) { + setIdentifierLocation := func(identifier ast.Identifier, identifierToken lexer.Token) { location = common.IdentifierLocation(identifier.Identifier) locationPos = identifier.Pos endPos = identifier.EndPosition(p.memoryGauge) + trailingComments = identifierToken.Comments.Trailing } parseLocation := func() error { @@ -700,7 +703,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { case lexer.TokenIdentifier: identifier := p.tokenToIdentifier(p.current) - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, p.current) p.next() default: @@ -787,7 +790,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return nil } - maybeParseFromIdentifier := func(identifier ast.Identifier) error { + maybeParseFromIdentifier := func(identifier ast.Identifier, identifierToken lexer.Token) error { // The current identifier is maybe the `from` keyword, // in which case the given (previous) identifier was // an imported identifier and not the import location. @@ -805,7 +808,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return err } } else { - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) } return nil @@ -819,6 +822,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { parseStringOrAddressLocation() case lexer.TokenIdentifier: + identifierToken := p.current identifier := p.tokenToIdentifier(p.current) // Skip the identifier p.nextSemanticToken() @@ -833,13 +837,13 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { return nil, err } case lexer.TokenIdentifier: - err := maybeParseFromIdentifier(identifier) + err := maybeParseFromIdentifier(identifier, identifierToken) if err != nil { return nil, err } case lexer.TokenEOF: // The previous identifier is the identifier location - setIdentifierLocation(identifier) + setIdentifierLocation(identifier, identifierToken) default: return nil, p.syntaxError( @@ -866,10 +870,14 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { location, ast.NewRange( p.memoryGauge, - startPosition, + startToken.StartPos, endPos, ), locationPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: trailingComments, + }, ), nil } diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 9c6d3ea19a..08307f7869 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -2318,6 +2318,39 @@ func TestParseImportDeclaration(t *testing.T) { ) }) + t.Run("no identifiers, string location, leading/trailing comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before foo +import "foo" /* After foo */`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.ImportDeclaration{ + Identifiers: nil, + Location: common.StringLocation("foo"), + LocationPos: ast.Position{Line: 3, Column: 7, Offset: 22}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 15}, + EndPos: ast.Position{Line: 3, Column: 11, Offset: 26}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After foo */")), + }, + }, + }, + }, + result, + ) + }) + t.Run("no identifiers, address location", func(t *testing.T) { t.Parallel() @@ -2455,11 +2488,11 @@ func TestParseImportDeclaration(t *testing.T) { ) }) - t.Run("three identifiers, address location", func(t *testing.T) { + t.Run("three identifiers, address location, trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo , bar , baz from 0x42`) + result, errs := testParseDeclarations(` import foo , bar , baz from 0x42 // After address`) require.Empty(t, errs) utils.AssertEqualWithDiff(t, @@ -2487,6 +2520,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 32, Offset: 32}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After address")), + }, + }, }, }, result, @@ -2534,11 +2572,11 @@ func TestParseImportDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, expected, result) }) - t.Run("no identifiers, identifier location", func(t *testing.T) { + t.Run("no identifiers, identifier location, trailing comment", func(t *testing.T) { t.Parallel() - result, errs := testParseDeclarations(` import foo`) + result, errs := testParseDeclarations(` import foo // After foo`) require.Empty(t, errs) utils.AssertEqualWithDiff(t, @@ -2551,6 +2589,11 @@ func TestParseImportDeclaration(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 10, Offset: 10}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After foo")), + }, + }, }, }, result, @@ -2570,6 +2613,7 @@ func TestParseImportDeclaration(t *testing.T) { }, errs) }) + t.Run("from keyword as second identifier", func(t *testing.T) { t.Parallel() From b9bd76a34fd6a8cccb0445330af5efbcb3f0f376 Mon Sep 17 00:00:00 2001 From: Bart Date: Fri, 27 Sep 2024 11:56:43 +0200 Subject: [PATCH 31/84] update event parsing tests --- runtime/parser/declaration_test.go | 120 ++++++++++++++++++++++++++++- 1 file changed, 117 insertions(+), 3 deletions(-) diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 08307f7869..24fefe10ad 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -2318,7 +2318,7 @@ func TestParseImportDeclaration(t *testing.T) { ) }) - t.Run("no identifiers, string location, leading/trailing comments", func(t *testing.T) { + t.Run("no identifiers, string location, with leading/trailing comments", func(t *testing.T) { t.Parallel() @@ -2488,7 +2488,7 @@ import "foo" /* After foo */`) ) }) - t.Run("three identifiers, address location, trailing comment", func(t *testing.T) { + t.Run("three identifiers, address location, with trailing comment", func(t *testing.T) { t.Parallel() @@ -2572,7 +2572,7 @@ import "foo" /* After foo */`) utils.AssertEqualWithDiff(t, expected, result) }) - t.Run("no identifiers, identifier location, trailing comment", func(t *testing.T) { + t.Run("no identifiers, identifier location, with trailing comment", func(t *testing.T) { t.Parallel() @@ -2911,6 +2911,120 @@ event E() // After E ) }) + t.Run("one parameter with comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(`event E2 ( + // Before a + a: Int // After a +)`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 28, + Line: 3, + Column: 4, + }, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After a")), + }, + }, + }, + StartPos: ast.Position{ + Offset: 28, + Line: 3, + Column: 4, + }, + }, + Identifier: ast.Identifier{ + Identifier: "a", + Pos: ast.Position{ + Offset: 25, + Line: 3, + Column: 1, + }, + }, + StartPos: ast.Position{ + Offset: 25, + Line: 3, + Column: 1, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before a")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 9, + Line: 1, + Column: 9, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 4, + Column: 0, + }, + }, + }, + StartPos: ast.Position{ + Offset: 9, + Line: 1, + Column: 9, + }, + Access: ast.AccessNotSpecified, + }, + Kind: common.DeclarationKindInitializer, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "E2", + Pos: ast.Position{ + Offset: 6, + Line: 1, + Column: 6, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 0, + Line: 1, + Column: 0, + }, + EndPos: ast.Position{ + Offset: 43, + Line: 4, + Column: 0, + }, + }, + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindEvent, + }, + }, + result, + ) + }) + t.Run("default event", func(t *testing.T) { t.Parallel() From f65f9d52a375595026cf11383930bc0b00baaefc Mon Sep 17 00:00:00 2001 From: Bart Date: Sat, 28 Sep 2024 22:55:50 +0200 Subject: [PATCH 32/84] fix old parser declaration tests --- runtime/old_parser/declaration_test.go | 40 +++++++++----------------- tools/storage-explorer/go.mod | 2 +- tools/storage-explorer/go.sum | 1 + 3 files changed, 16 insertions(+), 27 deletions(-) diff --git a/runtime/old_parser/declaration_test.go b/runtime/old_parser/declaration_test.go index a009603a0d..b8ef72a441 100644 --- a/runtime/old_parser/declaration_test.go +++ b/runtime/old_parser/declaration_test.go @@ -817,7 +817,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, }, StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, @@ -857,7 +857,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// First line")), ast.NewComment(nil, []byte("/// Second line")), }, @@ -900,7 +900,7 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), }, }, @@ -6883,7 +6883,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// noReturnNoBlock")), }, }, @@ -6902,7 +6902,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// returnNoBlock")), }, }, @@ -6931,7 +6931,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, Comments: ast.Comments{ - Leading: []ast.Comment{ + Leading: []*ast.Comment{ ast.NewComment(nil, []byte("/// returnAndBlock")), }, }, @@ -6962,8 +6962,8 @@ func TestParseMemberDocStrings(t *testing.T) { EndPos: ast.Position{Offset: 246, Line: 11, Column: 44}, }, Comments: ast.Comments{ - Leading: []ast.Comment{}, - Trailing: []ast.Comment{}, + Leading: []*ast.Comment{}, + Trailing: []*ast.Comment{}, }, }, }, @@ -7015,12 +7015,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []ast.Comment{ - ast.NewComment(nil, []byte("/// unknown")), - }, - }, + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7037,12 +7033,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []ast.Comment{ - ast.NewComment(nil, []byte("/// initNoBlock")), - }, - }, + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7059,12 +7051,8 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []ast.Comment{ - ast.NewComment(nil, []byte("/// destroyWithBlock")), - }, - }, + Access: ast.AccessNotSpecified, + Comments: ast.Comments{}, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/tools/storage-explorer/go.mod b/tools/storage-explorer/go.mod index e29530edb8..676751b7c2 100644 --- a/tools/storage-explorer/go.mod +++ b/tools/storage-explorer/go.mod @@ -4,7 +4,7 @@ go 1.22 require ( github.com/gorilla/mux v1.8.1 - github.com/onflow/atree v0.8.0-rc.6 + github.com/onflow/atree v0.8.0 github.com/onflow/cadence v1.0.0-preview.52 github.com/onflow/flow-go v0.37.10 github.com/rs/zerolog v1.32.0 diff --git a/tools/storage-explorer/go.sum b/tools/storage-explorer/go.sum index 54aea1efc8..40fa2d4f2b 100644 --- a/tools/storage-explorer/go.sum +++ b/tools/storage-explorer/go.sum @@ -1910,6 +1910,7 @@ github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6 github.com/onflow/atree v0.6.1-0.20230711151834-86040b30171f/go.mod h1:xvP61FoOs95K7IYdIYRnNcYQGf4nbF/uuJ0tHf4DRuM= github.com/onflow/atree v0.8.0-rc.6 h1:GWgaylK24b5ta2Hq+TvyOF7X5tZLiLzMMn7lEt59fsA= github.com/onflow/atree v0.8.0-rc.6/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= +github.com/onflow/atree v0.8.0/go.mod h1:yccR+LR7xc1Jdic0mrjocbHvUD7lnVvg8/Ct1AA5zBo= github.com/onflow/crypto v0.25.0/go.mod h1:C8FbaX0x8y+FxWjbkHy0Q4EASCDR9bSPWZqlpCLYyVI= github.com/onflow/crypto v0.25.2 h1:GjHunqVt+vPcdqhxxhAXiMIF3YiLX7gTuTR5O+VG2ns= github.com/onflow/crypto v0.25.2/go.mod h1:fY7eLqUdMKV8EGOw301unP8h7PvLVy8/6gVR++/g0BY= From a1455522bab66f3d55e4f8223d4fbfc5bf99bb0c Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 29 Sep 2024 13:27:59 +0200 Subject: [PATCH 33/84] update parsing for composite declaration, declaration field, declaration function --- runtime/ast/composite.go | 43 +-- runtime/ast/composite_test.go | 8 +- runtime/ast/interface.go | 16 +- runtime/ast/interface_test.go | 16 +- runtime/ast/variable_declaration.go | 10 +- runtime/ast/variable_declaration_test.go | 6 +- runtime/old_parser/declaration.go | 6 +- runtime/parser/declaration.go | 89 +++-- runtime/parser/declaration_test.go | 368 +++++++++++++++++++- runtime/parser/function.go | 7 +- runtime/parser/transaction.go | 1 + runtime/sema/check_composite_declaration.go | 4 +- runtime/sema/check_interface_declaration.go | 2 +- runtime/stdlib/test.go | 2 +- 14 files changed, 492 insertions(+), 86 deletions(-) diff --git a/runtime/ast/composite.go b/runtime/ast/composite.go index d6e74fec43..e092c0fe80 100644 --- a/runtime/ast/composite.go +++ b/runtime/ast/composite.go @@ -51,7 +51,6 @@ func IsResourceDestructionDefaultEvent(identifier string) bool { type CompositeDeclaration struct { Members *Members - DocString string Conformances []*NominalType Identifier Identifier Range @@ -72,7 +71,6 @@ func NewCompositeDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declarationRange Range, comments Comments, ) *CompositeDeclaration { @@ -84,7 +82,6 @@ func NewCompositeDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, Comments: comments, } @@ -123,17 +120,19 @@ func (d *CompositeDeclaration) DeclarationMembers() *Members { } func (d *CompositeDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *CompositeDeclaration) MarshalJSON() ([]byte, error) { type Alias CompositeDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "CompositeDeclaration", - Alias: (*Alias)(d), + Type: "CompositeDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -307,12 +306,14 @@ const ( type FieldDeclaration struct { TypeAnnotation *TypeAnnotation - DocString string - Identifier Identifier + // TODO(preserve-comments): Remove + DocString string + Identifier Identifier Range Access Access VariableKind VariableKind Flags FieldDeclarationFlags + Comments } var _ Element = &FieldDeclaration{} @@ -326,8 +327,8 @@ func NewFieldDeclaration( variableKind VariableKind, identifier Identifier, typeAnnotation *TypeAnnotation, - docString string, declRange Range, + comments Comments, ) *FieldDeclaration { common.UseMemory(memoryGauge, common.FieldDeclarationMemoryUsage) @@ -345,8 +346,8 @@ func NewFieldDeclaration( VariableKind: variableKind, Identifier: identifier, TypeAnnotation: typeAnnotation, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -385,16 +386,18 @@ func (d *FieldDeclaration) MarshalJSON() ([]byte, error) { type Alias FieldDeclaration return json.Marshal(&struct { *Alias - Type string - Flags FieldDeclarationFlags `json:",omitempty"` - IsStatic bool - IsNative bool + Type string + Flags FieldDeclarationFlags `json:",omitempty"` + IsStatic bool + IsNative bool + DocString string }{ - Type: "FieldDeclaration", - Alias: (*Alias)(d), - IsStatic: d.IsStatic(), - IsNative: d.IsNative(), - Flags: 0, + Type: "FieldDeclaration", + Alias: (*Alias)(d), + IsStatic: d.IsStatic(), + IsNative: d.IsNative(), + Flags: 0, + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/composite_test.go b/runtime/ast/composite_test.go index 104e0cfa4e..f9e725a142 100644 --- a/runtime/ast/composite_test.go +++ b/runtime/ast/composite_test.go @@ -406,8 +406,12 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/interface.go b/runtime/ast/interface.go index 8fa4c622f1..b4565ed455 100644 --- a/runtime/ast/interface.go +++ b/runtime/ast/interface.go @@ -30,12 +30,12 @@ import ( type InterfaceDeclaration struct { Members *Members - DocString string Identifier Identifier Conformances []*NominalType Range Access Access CompositeKind common.CompositeKind + Comments } var _ Element = &InterfaceDeclaration{} @@ -49,8 +49,8 @@ func NewInterfaceDeclaration( identifier Identifier, conformances []*NominalType, members *Members, - docString string, declRange Range, + comments Comments, ) *InterfaceDeclaration { common.UseMemory(gauge, common.InterfaceDeclarationMemoryUsage) @@ -60,8 +60,8 @@ func NewInterfaceDeclaration( Identifier: identifier, Conformances: conformances, Members: members, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -96,17 +96,19 @@ func (d *InterfaceDeclaration) DeclarationMembers() *Members { } func (d *InterfaceDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *InterfaceDeclaration) MarshalJSON() ([]byte, error) { type Alias InterfaceDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "InterfaceDeclaration", - Alias: (*Alias)(d), + Type: "InterfaceDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/interface_test.go b/runtime/ast/interface_test.go index 24819d9c35..98888255ca 100644 --- a/runtime/ast/interface_test.go +++ b/runtime/ast/interface_test.go @@ -42,8 +42,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -95,8 +99,12 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { }, }, }, - Members: NewUnmeteredMembers([]Declaration{}), - DocString: "test", + Members: NewUnmeteredMembers([]Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/variable_declaration.go b/runtime/ast/variable_declaration.go index 43620b0289..a88dbc887a 100644 --- a/runtime/ast/variable_declaration.go +++ b/runtime/ast/variable_declaration.go @@ -228,12 +228,14 @@ func (d *VariableDeclaration) MarshalJSON() ([]byte, error) { type Alias VariableDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string Range }{ - Type: "VariableDeclaration", - Range: NewUnmeteredRangeFromPositioned(d), - Alias: (*Alias)(d), + Type: "VariableDeclaration", + Range: NewUnmeteredRangeFromPositioned(d), + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/variable_declaration_test.go b/runtime/ast/variable_declaration_test.go index c34430f631..ead6891cd0 100644 --- a/runtime/ast/variable_declaration_test.go +++ b/runtime/ast/variable_declaration_test.go @@ -71,7 +71,11 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { EndPos: Position{Offset: 28, Line: 29, Column: 30}, }, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, } actual, err := json.Marshal(decl) diff --git a/runtime/old_parser/declaration.go b/runtime/old_parser/declaration.go index 80276688c9..517f5c0c4a 100644 --- a/runtime/old_parser/declaration.go +++ b/runtime/old_parser/declaration.go @@ -820,7 +820,6 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, startPos, @@ -912,12 +911,12 @@ func parseFieldWithVariableKind( variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } @@ -1061,7 +1060,6 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, ast.Comments{}, ), nil @@ -1468,12 +1466,12 @@ func parseFieldDeclarationWithoutVariableKind( ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 6285b25a11..b64acf9c32 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -64,6 +64,7 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] func parseDeclaration(p *parser) (ast.Declaration, error) { // TODO(preserve-comments): Refactor & remove var docString string + var startComments []*ast.Comment var access ast.Access = ast.AccessNotSpecified var accessPos *ast.Position @@ -114,6 +115,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { purityPos, staticPos, nativePos, + startComments, ) case KeywordImport: @@ -134,7 +136,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for event") } - return parseEventDeclaration(p, access, accessPos, docString) + return parseEventDeclaration(p, access, accessPos, startComments) case KeywordStruct: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindStructure) @@ -144,7 +146,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for struct") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordResource: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindResource) @@ -154,7 +156,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for resource") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEntitlement: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) @@ -181,7 +183,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for contract") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEnum: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEnum) @@ -191,7 +193,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for enum") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordTransaction: err := rejectAllModifiers(p, access, accessPos, staticPos, nativePos, common.DeclarationKindTransaction) @@ -238,7 +240,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { pos := p.current.StartPos accessPos = &pos var err error - access, err = parseAccess(p) + access, startComments, err = parseAccess(p) if err != nil { return nil, err } @@ -420,22 +422,22 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { // : 'access(self)' // | 'access(all)' ( '(' 'set' ')' )? // | 'access' '(' ( 'self' | 'contract' | 'account' | 'all' | entitlementList ) ')' -func parseAccess(p *parser) (ast.Access, error) { - +func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { switch string(p.currentTokenSource()) { case KeywordAccess: + accessTokenComments := p.current.Comments.PackToList() // Skip the `access` keyword p.nextSemanticToken() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { - return ast.AccessNotSpecified, p.syntaxError( + return ast.AccessNotSpecified, accessTokenComments, p.syntaxError( "expected keyword %s, got %s", enumeratedAccessModifierKeywords, p.current.Type, @@ -474,7 +476,7 @@ func parseAccess(p *parser) (ast.Access, error) { entitlementMapName, err := parseNominalType(p, lowestBindingPower) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } access = ast.NewMappedAccess(entitlementMapName, keywordPos) @@ -483,20 +485,20 @@ func parseAccess(p *parser) (ast.Access, error) { default: entitlements, err := parseEntitlementList(p) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } access = ast.NewEntitlementAccess(entitlements) } _, err = p.mustOne(lexer.TokenParenClose) if err != nil { - return ast.AccessNotSpecified, err + return ast.AccessNotSpecified, accessTokenComments, err } - return access, nil + return access, accessTokenComments, nil default: - return ast.AccessNotSpecified, errors.NewUnreachableError() + return ast.AccessNotSpecified, nil, errors.NewUnreachableError() } } @@ -940,7 +942,7 @@ func parseEventDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (*ast.CompositeDeclaration, error) { startToken := p.current startPos := startToken.StartPos @@ -991,6 +993,10 @@ func parseEventDeclaration( }, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewCompositeDeclaration( p.memoryGauge, access, @@ -998,14 +1004,13 @@ func parseEventDeclaration( identifier, nil, members, - docString, ast.NewRange( p.memoryGauge, startPos, parameterList.EndPos, ), ast.Comments{ - Leading: startToken.Comments.Leading, + Leading: leadingComments, }, ), nil } @@ -1045,10 +1050,8 @@ func parseFieldWithVariableKind( accessPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, + startComments []*ast.Comment, ) (*ast.FieldDeclaration, error) { - // TODO(preserve-comments): Implement - var docString string - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) var variableKind ast.VariableKind @@ -1093,12 +1096,14 @@ func parseFieldWithVariableKind( variableKind, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{ + Leading: startComments, + }, ), nil } @@ -1324,10 +1329,11 @@ func parseCompositeOrInterfaceDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1407,6 +1413,10 @@ func parseCompositeOrInterfaceDeclaration( endToken.EndPos, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + if isInterface { return ast.NewInterfaceDeclaration( p.memoryGauge, @@ -1415,8 +1425,10 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { return ast.NewCompositeDeclaration( @@ -1426,9 +1438,10 @@ func parseCompositeOrInterfaceDeclaration( identifier, conformances, members, - docString, declarationRange, - ast.Comments{}, + ast.Comments{ + Leading: leadingComments, + }, ), nil } } @@ -1577,8 +1590,9 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | enumCase // | pragmaDeclaration func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { - // TODO(preserve-comments): Implement + // TODO(preserve-comments): Remove var docString string + var startComments []*ast.Comment const functionBlockIsOptional = true @@ -1623,6 +1637,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { accessPos, staticPos, nativePos, + startComments, ) case KeywordCase: @@ -1645,6 +1660,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { purityPos, staticPos, nativePos, + startComments, ) case KeywordEvent: @@ -1655,7 +1671,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseEventDeclaration(p, access, accessPos, docString) + return parseEventDeclaration(p, access, accessPos, startComments) case KeywordStruct: if purity != ast.FunctionPurityUnspecified { @@ -1665,7 +1681,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordResource: if purity != ast.FunctionPurityUnspecified { @@ -1675,7 +1691,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordContract: if purity != ast.FunctionPurityUnspecified { @@ -1685,7 +1701,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordEntitlement: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) @@ -1705,7 +1721,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, docString) + return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordAttachment: return parseAttachmentDeclaration(p, access, accessPos, docString) @@ -1743,7 +1759,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { pos := p.current.StartPos accessPos = &pos var err error - access, err = parseAccess(p) + access, startComments, err = parseAccess(p) if err != nil { return nil, err } @@ -1881,9 +1897,6 @@ func parseFieldDeclarationWithoutVariableKind( nativePos *ast.Position, identifier ast.Identifier, ) (*ast.FieldDeclaration, error) { - // TODO(preserve-comments): Implement - var docString string - startPos := ast.EarliestPosition(identifier.Pos, accessPos, staticPos, nativePos) _, err := p.mustOne(lexer.TokenColon) @@ -1906,12 +1919,12 @@ func parseFieldDeclarationWithoutVariableKind( ast.VariableKindNotSpecified, identifier, typeAnnotation, - docString, ast.NewRange( p.memoryGauge, startPos, typeAnnotation.EndPosition(p.memoryGauge), ), + ast.Comments{}, ), nil } diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 24fefe10ad..a5975d4d84 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -1873,7 +1873,10 @@ func TestParseAccess(t *testing.T) { return Parse( nil, []byte(input), - parseAccess, + func(p *parser) (ast.Access, error) { + access, _, err := parseAccess(p) + return access, err + }, Config{}, ) } @@ -3196,6 +3199,7 @@ func TestParseFieldWithVariableKind(t *testing.T) { nil, nil, nil, + nil, ) }, Config{}, @@ -3981,6 +3985,368 @@ func TestParseCompositeDeclaration(t *testing.T) { ) }) + t.Run("struct, with fields, functions, special functions, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before Test + struct Test { + // Before foo + access(all) var foo: Int + + // Before init + init(foo: Int) { + self.foo = foo + } + + // Before getFoo + access(all) fun getFoo(): Int { + return self.foo + } + } + `) + + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.CompositeDeclaration{ + Members: ast.NewUnmeteredMembers( + []ast.Declaration{ + &ast.FieldDeclaration{ + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, + }, + }, + StartPos: ast.Position{ + Offset: 97, + Line: 5, + Column: 31, + }, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 92, + Line: 5, + Column: 26, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 76, + Line: 5, + Column: 10, + }, + EndPos: ast.Position{ + Offset: 99, + Line: 5, + Column: 33, + }, + }, + Access: ast.AccessAll, + VariableKind: 0x1, + Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before foo")), + }, + }, + }, + &ast.SpecialFunctionDeclaration{ + FunctionDeclaration: &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Parameters: []*ast.Parameter{ + { + TypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, + }, + }, + StartPos: ast.Position{ + Offset: 147, + Line: 8, + Column: 20, + }, + IsResource: false, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 142, + Line: 8, + Column: 15, + }, + }, + StartPos: ast.Position{ + Offset: 142, + Line: 8, + Column: 15, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 141, + Line: 8, + Column: 14, + }, + EndPos: ast.Position{ + Offset: 150, + Line: 8, + Column: 23, + }, + }, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.AssignmentStatement{ + Target: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{ + Offset: 168, + Line: 9, + Column: 14, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 173, + Line: 9, + Column: 19, + }, + }, + AccessPos: ast.Position{ + Offset: 172, + Line: 9, + Column: 18, + }, + Optional: false, + }, + Transfer: &ast.Transfer{ + Operation: 0x1, + Pos: ast.Position{ + Offset: 177, + Line: 9, + Column: 23, + }, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 179, + Line: 9, + Column: 25, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 152, + Line: 8, + Column: 25, + }, + EndPos: ast.Position{ + Offset: 193, + Line: 10, + Column: 10, + }, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "init", + Pos: ast.Position{ + Offset: 137, + Line: 8, + Column: 10, + }, + }, + StartPos: ast.Position{ + Offset: 137, + Line: 8, + Column: 10, + }, + Access: ast.AccessNotSpecified, + Flags: 0x00, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before init")), + }, + }, + }, + Kind: 0xd, + }, + &ast.FunctionDeclaration{ + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 255, + Line: 13, + Column: 32, + }, + EndPos: ast.Position{ + Offset: 256, + Line: 13, + Column: 33, + }, + }, + }, + ReturnTypeAnnotation: &ast.TypeAnnotation{ + Type: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "Int", + Pos: ast.Position{ + Offset: 259, + Line: 13, + Column: 36, + }, + }, + }, + StartPos: ast.Position{ + Offset: 259, + Line: 13, + Column: 36, + }, + IsResource: false, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.MemberExpression{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "self", + Pos: ast.Position{ + Offset: 286, + Line: 14, + Column: 21, + }, + }, + }, + Identifier: ast.Identifier{ + Identifier: "foo", + Pos: ast.Position{ + Offset: 291, + Line: 14, + Column: 26, + }, + }, + AccessPos: ast.Position{ + Offset: 290, + Line: 14, + Column: 25, + }, + Optional: false, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 279, + Line: 14, + Column: 14, + }, + EndPos: ast.Position{ + Offset: 293, + Line: 14, + Column: 28, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 263, + Line: 13, + Column: 40, + }, + EndPos: ast.Position{ + Offset: 305, + Line: 15, + Column: 10, + }, + }, + }, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before getFoo")), + }, + }, + Identifier: ast.Identifier{ + Identifier: "getFoo", + Pos: ast.Position{ + Offset: 249, + Line: 13, + Column: 26, + }, + }, + StartPos: ast.Position{ + Offset: 233, + Line: 13, + Column: 10, + }, + Access: ast.AccessAll, + Flags: 0x00, + }, + }, + ), + Identifier: ast.Identifier{ + Identifier: "Test", + Pos: ast.Position{ + Offset: 35, + Line: 3, + Column: 13, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 28, + Line: 3, + Column: 6, + }, + EndPos: ast.Position{ + Offset: 313, + Line: 16, + Column: 6, + }, + }, + Access: ast.AccessNotSpecified, + CompositeKind: common.CompositeKindStructure, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before Test")), + }, + }, + }, + }, + result, + ) + }) + t.Run("struct with view member", func(t *testing.T) { t.Parallel() diff --git a/runtime/parser/function.go b/runtime/parser/function.go index e10325d152..4ca302cc37 100644 --- a/runtime/parser/function.go +++ b/runtime/parser/function.go @@ -355,6 +355,7 @@ func parseFunctionDeclaration( purityPos *ast.Position, staticPos *ast.Position, nativePos *ast.Position, + startComments []*ast.Comment, ) (*ast.FunctionDeclaration, error) { startToken := p.current startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) @@ -388,6 +389,10 @@ func parseFunctionDeclaration( return nil, err } + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewFunctionDeclarationWithComments( p.memoryGauge, access, @@ -401,7 +406,7 @@ func parseFunctionDeclaration( functionBlock, startPos, ast.Comments{ - Leading: startToken.Leading, + Leading: leadingComments, }, ), nil } diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index d5ebfd23b6..b2ce069287 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -237,6 +237,7 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro nil, nil, nil, + nil, ) if err != nil { return nil, err diff --git a/runtime/sema/check_composite_declaration.go b/runtime/sema/check_composite_declaration.go index af0d07d715..1585a16c1f 100644 --- a/runtime/sema/check_composite_declaration.go +++ b/runtime/sema/check_composite_declaration.go @@ -1013,7 +1013,7 @@ func (checker *Checker) declareContractValue( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: compositeType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: contracts are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindContract, @@ -1085,7 +1085,7 @@ func (checker *Checker) declareEnumConstructor( _, err := checker.valueActivations.declare(variableDeclaration{ identifier: declaration.Identifier.Identifier, ty: constructorType, - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), // NOTE: enums are always public access: PrimitiveAccess(ast.AccessAll), kind: common.DeclarationKindEnum, diff --git a/runtime/sema/check_interface_declaration.go b/runtime/sema/check_interface_declaration.go index bfef6c7420..7a64af65aa 100644 --- a/runtime/sema/check_interface_declaration.go +++ b/runtime/sema/check_interface_declaration.go @@ -286,7 +286,7 @@ func (checker *Checker) declareInterfaceType(declaration *ast.InterfaceDeclarati ty: interfaceType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) checker.report(err) diff --git a/runtime/stdlib/test.go b/runtime/stdlib/test.go index eb65c13985..86fa745907 100644 --- a/runtime/stdlib/test.go +++ b/runtime/stdlib/test.go @@ -476,7 +476,7 @@ func TestCheckerContractValueHandler( return StandardLibraryValue{ Name: declaration.Identifier.Identifier, Type: constructorType, - DocString: declaration.DocString, + DocString: declaration.DeclarationDocString(), Kind: declaration.DeclarationKind(), Position: &declaration.Identifier.Pos, ArgumentLabels: constructorArgumentLabels, From 39457a81d0613b24711978dceed4e9f2ba54e234 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 29 Sep 2024 13:45:12 +0200 Subject: [PATCH 34/84] update attachment declaration parsing --- runtime/ast/attachment.go | 14 ++++++----- runtime/ast/attachment_test.go | 16 +++++++++--- runtime/parser/declaration.go | 18 ++++++++++---- runtime/parser/declaration_test.go | 39 ++++++++++++++++++++++++++++++ 4 files changed, 72 insertions(+), 15 deletions(-) diff --git a/runtime/ast/attachment.go b/runtime/ast/attachment.go index 32cd047473..ba21b854e3 100644 --- a/runtime/ast/attachment.go +++ b/runtime/ast/attachment.go @@ -34,8 +34,8 @@ type AttachmentDeclaration struct { BaseType *NominalType Conformances []*NominalType Members *Members - DocString string Range + Comments } var _ Element = &AttachmentDeclaration{} @@ -50,8 +50,8 @@ func NewAttachmentDeclaration( baseType *NominalType, conformances []*NominalType, members *Members, - docString string, declarationRange Range, + comments Comments, ) *AttachmentDeclaration { common.UseMemory(memoryGauge, common.AttachmentDeclarationMemoryUsage) @@ -61,8 +61,8 @@ func NewAttachmentDeclaration( BaseType: baseType, Conformances: conformances, Members: members, - DocString: docString, Range: declarationRange, + Comments: comments, } } @@ -99,7 +99,7 @@ func (d *AttachmentDeclaration) DeclarationMembers() *Members { } func (d *AttachmentDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (*AttachmentDeclaration) Kind() common.CompositeKind { @@ -199,9 +199,11 @@ func (d *AttachmentDeclaration) MarshalJSON() ([]byte, error) { return json.Marshal(&struct { Type string *Alias + DocString string }{ - Type: "AttachmentDeclaration", - Alias: (*Alias)(d), + Type: "AttachmentDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/attachment_test.go b/runtime/ast/attachment_test.go index d57c61e365..7413504078 100644 --- a/runtime/ast/attachment_test.go +++ b/runtime/ast/attachment_test.go @@ -56,8 +56,12 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ), }, }, - Members: NewMembers(nil, []Declaration{}), - DocString: "test", + Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 1, Line: 2, Column: 3}, EndPos: Position{Offset: 4, Line: 5, Column: 6}, @@ -141,8 +145,12 @@ func TestAttachmentDeclaration_Doc(t *testing.T) { ), }, }, - Members: NewMembers(nil, []Declaration{}), - DocString: "test", + Members: NewMembers(nil, []Declaration{}), + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 1, Line: 2, Column: 3}, EndPos: Position{Offset: 4, Line: 5, Column: 6}, diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index b64acf9c32..038853dbd3 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -173,7 +173,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if err != nil { return nil, err } - return parseAttachmentDeclaration(p, access, accessPos, docString) + return parseAttachmentDeclaration(p, access, accessPos, startComments) case KeywordContract: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindContract) @@ -1450,9 +1450,11 @@ func parseAttachmentDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1529,6 +1531,10 @@ func parseAttachmentDeclaration( endToken.EndPos, ) + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewAttachmentDeclaration( p.memoryGauge, access, @@ -1536,8 +1542,10 @@ func parseAttachmentDeclaration( baseNominalType, conformances, members, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } @@ -1724,7 +1732,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) case KeywordAttachment: - return parseAttachmentDeclaration(p, access, accessPos, docString) + return parseAttachmentDeclaration(p, access, accessPos, startComments) case KeywordView: if purity != ast.FunctionPurityUnspecified { diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index a5975d4d84..b7e3877410 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -4573,6 +4573,45 @@ func TestParseAttachmentDeclaration(t *testing.T) { ) }) + t.Run("no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before E + access(all) attachment E for S {}`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.AttachmentDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "E", + Pos: ast.Position{Line: 3, Column: 27, Offset: 44}, + }, + BaseType: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 33, Offset: 50}, + }, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 4, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 36, Offset: 53}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before E")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested in contract", func(t *testing.T) { t.Parallel() From 1c660166f3d1c3d99a265dd9681a0686aa110e9d Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 29 Sep 2024 14:05:26 +0200 Subject: [PATCH 35/84] update interface declaration, transaction declaration parsing --- runtime/ast/transaction_declaration.go | 13 ++-- runtime/ast/transaction_declaration_test.go | 8 ++- runtime/old_parser/transaction.go | 2 +- runtime/parser/declaration.go | 2 +- runtime/parser/declaration_test.go | 66 +++++++++++++++++++++ runtime/parser/transaction.go | 10 +++- 6 files changed, 90 insertions(+), 11 deletions(-) diff --git a/runtime/ast/transaction_declaration.go b/runtime/ast/transaction_declaration.go index a9b6d91b23..17cefee9c6 100644 --- a/runtime/ast/transaction_declaration.go +++ b/runtime/ast/transaction_declaration.go @@ -35,6 +35,7 @@ type TransactionDeclaration struct { DocString string Fields []*FieldDeclaration Range + Comments } var _ Element = &TransactionDeclaration{} @@ -49,8 +50,8 @@ func NewTransactionDeclaration( preConditions *Conditions, postConditions *Conditions, execute *SpecialFunctionDeclaration, - docString string, declRange Range, + comments Comments, ) *TransactionDeclaration { common.UseMemory(gauge, common.TransactionDeclarationMemoryUsage) @@ -61,8 +62,8 @@ func NewTransactionDeclaration( PreConditions: preConditions, PostConditions: postConditions, Execute: execute, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -111,10 +112,12 @@ func (d *TransactionDeclaration) MarshalJSON() ([]byte, error) { type Alias TransactionDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "TransactionDeclaration", - Alias: (*Alias)(d), + Type: "TransactionDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/transaction_declaration_test.go b/runtime/ast/transaction_declaration_test.go index 44f4afd39c..f73ccf106e 100644 --- a/runtime/ast/transaction_declaration_test.go +++ b/runtime/ast/transaction_declaration_test.go @@ -45,8 +45,12 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { Prepare: nil, PreConditions: &Conditions{}, PostConditions: &Conditions{}, - DocString: "test", - Execute: nil, + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, + Execute: nil, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/old_parser/transaction.go b/runtime/old_parser/transaction.go index a6c72419aa..188de3a1d7 100644 --- a/runtime/old_parser/transaction.go +++ b/runtime/old_parser/transaction.go @@ -202,12 +202,12 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, startPos, endPos, ), + ast.Comments{}, ), nil } diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 038853dbd3..766d930757 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -204,7 +204,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { return nil, NewSyntaxError(*purityPos, "invalid view modifier for transaction") } - return parseTransactionDeclaration(p, docString) + return parseTransactionDeclaration(p, startComments) case KeywordView: if purity != ast.FunctionPurityUnspecified { diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index b7e3877410..122595cac5 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -5159,6 +5159,40 @@ func TestParseInterfaceDeclaration(t *testing.T) { ) }) + t.Run("struct, no conformances, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` +// Before S +access(all) struct interface S { }`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.InterfaceDeclaration{ + Access: ast.AccessAll, + CompositeKind: common.CompositeKindStructure, + Identifier: ast.Identifier{ + Identifier: "S", + Pos: ast.Position{Line: 3, Column: 29, Offset: 42}, + }, + Members: &ast.Members{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 13}, + EndPos: ast.Position{Line: 3, Column: 33, Offset: 46}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before S")), + }, + }, + }, + }, + result, + ) + }) + t.Run("struct, interface keyword as name", func(t *testing.T) { t.Parallel() @@ -5739,6 +5773,38 @@ func TestParseTransactionDeclaration(t *testing.T) { ) }) + t.Run("EmptyTransaction, comments", func(t *testing.T) { + + const code = ` + // Before tx + transaction {} + ` + result, errs := testParseProgram(code) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.TransactionDeclaration{ + Fields: nil, + Prepare: nil, + PreConditions: nil, + PostConditions: nil, + Execute: nil, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before tx")), + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 22, Line: 3, Column: 4}, + EndPos: ast.Position{Offset: 35, Line: 3, Column: 17}, + }, + }, + }, + result.Declarations(), + ) + }) + t.Run("SimpleTransaction", func(t *testing.T) { const code = ` transaction { diff --git a/runtime/parser/transaction.go b/runtime/parser/transaction.go index b2ce069287..2632d51c06 100644 --- a/runtime/parser/transaction.go +++ b/runtime/parser/transaction.go @@ -39,7 +39,7 @@ import ( // | /* no execute or postConditions */ // ) // '}' -func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionDeclaration, error) { +func parseTransactionDeclaration(p *parser, startComments []*ast.Comment) (*ast.TransactionDeclaration, error) { startToken := p.current startPos := startToken.StartPos @@ -196,6 +196,10 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD } } + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + return ast.NewTransactionDeclaration( p.memoryGauge, parameterList, @@ -204,12 +208,14 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD preConditions, postConditions, execute, - docString, ast.NewRange( p.memoryGauge, startPos, endPos, ), + ast.Comments{ + Leading: leadingComments, + }, ), nil } From 3575509fef4e313ed7191db11bc9bc58d8980cf5 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 29 Sep 2024 14:20:17 +0200 Subject: [PATCH 36/84] update entitlement parsing --- runtime/ast/entitlement_declaration.go | 32 +++++----- runtime/ast/entitlement_declaration_test.go | 24 +++++-- runtime/ast/transaction_declaration.go | 2 +- runtime/parser/declaration.go | 25 +++++--- runtime/parser/declaration_test.go | 70 +++++++++++++++++++-- runtime/sema/check_interface_declaration.go | 4 +- 6 files changed, 122 insertions(+), 35 deletions(-) diff --git a/runtime/ast/entitlement_declaration.go b/runtime/ast/entitlement_declaration.go index 6a53e4e445..de363dbe62 100644 --- a/runtime/ast/entitlement_declaration.go +++ b/runtime/ast/entitlement_declaration.go @@ -30,9 +30,9 @@ import ( type EntitlementDeclaration struct { Access Access - DocString string Identifier Identifier Range + Comments } var _ Element = &EntitlementDeclaration{} @@ -43,16 +43,16 @@ func NewEntitlementDeclaration( gauge common.MemoryGauge, access Access, identifier Identifier, - docString string, declRange Range, + comments Comments, ) *EntitlementDeclaration { common.UseMemory(gauge, common.EntitlementDeclarationMemoryUsage) return &EntitlementDeclaration{ Access: access, Identifier: identifier, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -85,17 +85,19 @@ func (d *EntitlementDeclaration) DeclarationMembers() *Members { } func (d *EntitlementDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } @@ -168,10 +170,10 @@ func (d *EntitlementMapRelation) Doc() prettier.Doc { // EntitlementMappingDeclaration type EntitlementMappingDeclaration struct { Access Access - DocString string Identifier Identifier Elements []EntitlementMapElement Range + Comments } var _ Element = &EntitlementMappingDeclaration{} @@ -183,8 +185,8 @@ func NewEntitlementMappingDeclaration( access Access, identifier Identifier, elements []EntitlementMapElement, - docString string, declRange Range, + comments Comments, ) *EntitlementMappingDeclaration { common.UseMemory(gauge, common.EntitlementMappingDeclarationMemoryUsage) @@ -192,8 +194,8 @@ func NewEntitlementMappingDeclaration( Access: access, Identifier: identifier, Elements: elements, - DocString: docString, Range: declRange, + Comments: comments, } } @@ -244,17 +246,19 @@ func (d *EntitlementMappingDeclaration) DeclarationMembers() *Members { } func (d *EntitlementMappingDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EntitlementMappingDeclaration) MarshalJSON() ([]byte, error) { type Alias EntitlementMappingDeclaration return json.Marshal(&struct { *Alias - Type string + Type string + DocString string }{ - Type: "EntitlementMappingDeclaration", - Alias: (*Alias)(d), + Type: "EntitlementMappingDeclaration", + Alias: (*Alias)(d), + DocString: d.DeclarationDocString(), }) } diff --git a/runtime/ast/entitlement_declaration_test.go b/runtime/ast/entitlement_declaration_test.go index b1801abfe4..d3ac5dbdac 100644 --- a/runtime/ast/entitlement_declaration_test.go +++ b/runtime/ast/entitlement_declaration_test.go @@ -37,7 +37,11 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -131,7 +135,11 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -227,7 +235,11 @@ func TestEntitlementMappingDeclaration_Doc(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -300,7 +312,11 @@ func TestEntitlementMappingDeclaration_String(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/runtime/ast/transaction_declaration.go b/runtime/ast/transaction_declaration.go index 17cefee9c6..32cb9ccad6 100644 --- a/runtime/ast/transaction_declaration.go +++ b/runtime/ast/transaction_declaration.go @@ -105,7 +105,7 @@ func (d *TransactionDeclaration) DeclarationMembers() *Members { } func (d *TransactionDeclaration) DeclarationDocString() string { - return "" + return d.Comments.LeadingDocString() } func (d *TransactionDeclaration) MarshalJSON() ([]byte, error) { diff --git a/runtime/parser/declaration.go b/runtime/parser/declaration.go index 766d930757..a57e9c1465 100644 --- a/runtime/parser/declaration.go +++ b/runtime/parser/declaration.go @@ -62,8 +62,6 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] } func parseDeclaration(p *parser) (ast.Declaration, error) { - // TODO(preserve-comments): Refactor & remove - var docString string var startComments []*ast.Comment var access ast.Access = ast.AccessNotSpecified @@ -166,7 +164,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) case KeywordAttachment: err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindAttachment) @@ -1207,9 +1205,10 @@ func parseEntitlementOrMappingDeclaration( p *parser, access ast.Access, accessPos *ast.Position, - docString string, + startComments []*ast.Comment, ) (ast.Declaration, error) { - startPos := p.current.StartPos + startToken := p.current + startPos := startToken.StartPos if accessPos != nil { startPos = *accessPos } @@ -1244,6 +1243,10 @@ func parseEntitlementOrMappingDeclaration( } p.nextSemanticToken() + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.Leading...) + if isMapping { _, err = p.mustOne(lexer.TokenBraceOpen) if err != nil { @@ -1271,8 +1274,10 @@ func parseEntitlementOrMappingDeclaration( access, identifier, elements, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { declarationRange := ast.NewRange( @@ -1285,8 +1290,10 @@ func parseEntitlementOrMappingDeclaration( p.memoryGauge, access, identifier, - docString, declarationRange, + ast.Comments{ + Leading: leadingComments, + }, ), nil } } @@ -1598,8 +1605,6 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | enumCase // | pragmaDeclaration func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { - // TODO(preserve-comments): Remove - var docString string var startComments []*ast.Comment const functionBlockIsOptional = true @@ -1719,7 +1724,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, docString) + return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) case KeywordEnum: if purity != ast.FunctionPurityUnspecified { diff --git a/runtime/parser/declaration_test.go b/runtime/parser/declaration_test.go index 122595cac5..e3e2acd85d 100644 --- a/runtime/parser/declaration_test.go +++ b/runtime/parser/declaration_test.go @@ -9640,6 +9640,38 @@ func TestParseEntitlementDeclaration(t *testing.T) { ) }) + t.Run("basic, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before ABC + access(all) entitlement ABC`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "ABC", + Pos: ast.Position{Line: 3, Column: 27, Offset: 45}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 21}, + EndPos: ast.Position{Line: 3, Column: 29, Offset: 47}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before ABC")), + }, + }, + }, + }, + result, + ) + }) + t.Run("nested entitlement", func(t *testing.T) { t.Parallel() @@ -9991,6 +10023,38 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { ) }) + t.Run("empty, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseDeclarations(` + // Before M + access(all) entitlement mapping M { }`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.EntitlementMappingDeclaration{ + Access: ast.AccessAll, + Identifier: ast.Identifier{ + Identifier: "M", + Pos: ast.Position{Line: 3, Column: 35, Offset: 51}, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 19}, + EndPos: ast.Position{Line: 3, Column: 39, Offset: 55}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before M")), + }, + }, + }, + }, + result, + ) + }) + t.Run("mappings", func(t *testing.T) { t.Parallel() @@ -10004,8 +10068,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{ @@ -10093,8 +10156,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { utils.AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ - Access: ast.AccessAll, - DocString: "", + Access: ast.AccessAll, Identifier: ast.Identifier{ Identifier: "M", Pos: ast.Position{ diff --git a/runtime/sema/check_interface_declaration.go b/runtime/sema/check_interface_declaration.go index 7a64af65aa..458e0a3025 100644 --- a/runtime/sema/check_interface_declaration.go +++ b/runtime/sema/check_interface_declaration.go @@ -485,7 +485,7 @@ func (checker *Checker) declareEntitlementType(declaration *ast.EntitlementDecla ty: entitlementType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) @@ -532,7 +532,7 @@ func (checker *Checker) declareEntitlementMappingType(declaration *ast.Entitleme ty: entitlementMapType, declarationKind: declaration.DeclarationKind(), access: checker.accessFromAstAccess(declaration.Access), - docString: declaration.DocString, + docString: declaration.DeclarationDocString(), allowOuterScopeShadowing: false, }) From 716a3f427df6682814e6077081bef44695db8459 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 6 Oct 2024 19:04:31 +0200 Subject: [PATCH 37/84] update return statement parsing --- runtime/ast/statement.go | 9 ++- runtime/old_parser/statement.go | 1 + runtime/parser/statement.go | 9 ++- runtime/parser/statement_test.go | 103 +++++++++++++++++++++++++++++++ 4 files changed, 120 insertions(+), 2 deletions(-) diff --git a/runtime/ast/statement.go b/runtime/ast/statement.go index e1e3b6c519..79520d3071 100644 --- a/runtime/ast/statement.go +++ b/runtime/ast/statement.go @@ -39,16 +39,23 @@ type Statement interface { type ReturnStatement struct { Expression Expression Range + Comments } var _ Element = &ReturnStatement{} var _ Statement = &ReturnStatement{} -func NewReturnStatement(gauge common.MemoryGauge, expression Expression, stmtRange Range) *ReturnStatement { +func NewReturnStatement( + gauge common.MemoryGauge, + expression Expression, + stmtRange Range, + comments Comments, +) *ReturnStatement { common.UseMemory(gauge, common.ReturnStatementMemoryUsage) return &ReturnStatement{ Expression: expression, Range: stmtRange, + Comments: comments, } } diff --git a/runtime/old_parser/statement.go b/runtime/old_parser/statement.go index c0c3bfc887..723f699ec5 100644 --- a/runtime/old_parser/statement.go +++ b/runtime/old_parser/statement.go @@ -250,6 +250,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + ast.Comments{}, ), nil } diff --git a/runtime/parser/statement.go b/runtime/parser/statement.go index 6697f94905..f803f79b27 100644 --- a/runtime/parser/statement.go +++ b/runtime/parser/statement.go @@ -247,7 +247,9 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( } func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { - tokenRange := p.current.Range + var endToken lexer.Token + startToken := p.current + tokenRange := startToken.Range endPosition := tokenRange.EndPos p.next() @@ -259,6 +261,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { var err error switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: + endToken = p.current break default: if !sawNewLine { @@ -279,6 +282,10 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: endToken.Comments.PackToList(), + }, ), nil } diff --git a/runtime/parser/statement_test.go b/runtime/parser/statement_test.go index 551760b676..37645da0d8 100644 --- a/runtime/parser/statement_test.go +++ b/runtime/parser/statement_test.go @@ -78,6 +78,66 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("no expression, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + + t.Run("no expression, semicolon, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return ; // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 5, Offset: 23}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on same line", func(t *testing.T) { t.Parallel() @@ -107,6 +167,49 @@ func TestParseReturnStatement(t *testing.T) { ) }) + t.Run("expression on same line, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before return +return 1 // After return +`) + require.Empty(t, errs) + + utils.AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Expression: &ast.IntegerExpression{ + PositiveLiteral: []byte("1"), + Value: big.NewInt(1), + Base: 10, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + // TODO(preserve-comments): Should we attach this to expression or statement node? + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + EndPos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + }, + }, + }, + result, + ) + }) + t.Run("expression on next line, no semicolon", func(t *testing.T) { t.Parallel() From de4bb911d02c64fd9ea0a16e7610458d5ee7f56c Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 11 May 2025 14:27:37 +0200 Subject: [PATCH 38/84] fix merge issues --- ast/comments.go | 2 +- go.mod | 2 +- parser/comment.go | 65 -------------------------------------- parser/declaration_test.go | 24 +++++++------- parser/expression.go | 2 +- parser/expression_test.go | 1 - parser/lexer/token.go | 5 +++ parser/parser.go | 2 -- parser/statement_test.go | 6 ++-- 9 files changed, 23 insertions(+), 86 deletions(-) delete mode 100644 parser/comment.go diff --git a/ast/comments.go b/ast/comments.go index 2daa66cde5..75822425e1 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -2,7 +2,7 @@ package ast import ( "bytes" - "github.com/onflow/cadence/runtime/common" + "github.com/onflow/cadence/common" "strings" ) diff --git a/go.mod b/go.mod index c279c9f537..589dec8646 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/cadence -go 1.23 +go 1.23.0 require ( github.com/bits-and-blooms/bitset v1.5.0 diff --git a/parser/comment.go b/parser/comment.go deleted file mode 100644 index 5510967d3e..0000000000 --- a/parser/comment.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package parser - -// TODO(merge): Do we need this file? -import ( - "github.com/onflow/cadence/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 775da640e6..d38393805a 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -83,7 +83,7 @@ var x = /* Before 1 */ 1 // After 1 `) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.VariableDeclaration{ Access: ast.AccessNotSpecified, @@ -532,7 +532,7 @@ func TestParseParameterList(t *testing.T) { @Int /* After type */ )`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, &ast.ParameterList{ Parameters: []*ast.Parameter{ { @@ -683,7 +683,7 @@ c /* After c identifier */ : Int // After param c )`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, &ast.ParameterList{ Parameters: []*ast.Parameter{ { @@ -2342,7 +2342,7 @@ func TestParseImportDeclaration(t *testing.T) { import "foo" /* After foo */`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.ImportDeclaration{ Identifiers: nil, @@ -2749,7 +2749,7 @@ event E() // After E `) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.CompositeDeclaration{ Access: ast.AccessNotSpecified, @@ -2936,7 +2936,7 @@ event E() // After E )`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.CompositeDeclaration{ Members: ast.NewUnmeteredMembers( @@ -4021,7 +4021,7 @@ func TestParseCompositeDeclaration(t *testing.T) { require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.CompositeDeclaration{ Members: ast.NewUnmeteredMembers( @@ -4594,7 +4594,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { access(all) attachment E for S {}`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.AttachmentDeclaration{ Access: ast.AccessAll, @@ -5180,7 +5180,7 @@ func TestParseInterfaceDeclaration(t *testing.T) { access(all) struct interface S { }`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.InterfaceDeclaration{ Access: ast.AccessAll, @@ -5794,7 +5794,7 @@ func TestParseTransactionDeclaration(t *testing.T) { result, errs := testParseProgram(code) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.TransactionDeclaration{ Fields: nil, @@ -9721,7 +9721,7 @@ func TestParseEntitlementDeclaration(t *testing.T) { access(all) entitlement ABC`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementDeclaration{ Access: ast.AccessAll, @@ -10104,7 +10104,7 @@ func TestParseEntitlementMappingDeclaration(t *testing.T) { access(all) entitlement mapping M { }`) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Declaration{ &ast.EntitlementMappingDeclaration{ Access: ast.AccessAll, diff --git a/parser/expression.go b/parser/expression.go index c1de72b1ee..09912834a5 100644 --- a/parser/expression.go +++ b/parser/expression.go @@ -1167,7 +1167,7 @@ func defineStringExpression() { literal = p.tokenSource(curToken) // remove quotation marks if they exist - if curToken == startToken { + if curToken.Equal(startToken) { literal = literal[1:] } diff --git a/parser/expression_test.go b/parser/expression_test.go index badcd80f68..4200603531 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) diff --git a/parser/lexer/token.go b/parser/lexer/token.go index c418da70cd..2d0e5dcf84 100644 --- a/parser/lexer/token.go +++ b/parser/lexer/token.go @@ -41,3 +41,8 @@ func (t Token) Is(ty TokenType) bool { func (t Token) Source(input []byte) []byte { return t.Range.Source(input) } + +func (t Token) Equal(other Token) bool { + // ignore comments, since they should not be treated as source code + return t.Type == other.Type && t.Range == other.Range && t.SpaceOrError == other.SpaceOrError +} diff --git a/parser/parser.go b/parser/parser.go index 3dd204a462..3b60ee6551 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -19,9 +19,7 @@ package parser import ( - "bytes" "os" - "strings" "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" diff --git a/parser/statement_test.go b/parser/statement_test.go index adf12a8ef4..f0071e6f6f 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -88,7 +88,7 @@ return // After return `) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Statement{ &ast.ReturnStatement{ Range: ast.Range{ @@ -117,7 +117,7 @@ return ; // After return `) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Statement{ &ast.ReturnStatement{ Range: ast.Range{ @@ -177,7 +177,7 @@ return 1 // After return `) require.Empty(t, errs) - utils.AssertEqualWithDiff(t, + AssertEqualWithDiff(t, []ast.Statement{ &ast.ReturnStatement{ Expression: &ast.IntegerExpression{ From 8f05adaee8a1b39b0b5623d74a12186b41f8bd93 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 11 May 2025 14:34:54 +0200 Subject: [PATCH 39/84] fix type errors --- old_parser/comment.go | 65 --------------------------------------- old_parser/declaration.go | 4 +-- 2 files changed, 2 insertions(+), 67 deletions(-) delete mode 100644 old_parser/comment.go diff --git a/old_parser/comment.go b/old_parser/comment.go deleted file mode 100644 index 13e2bc931f..0000000000 --- a/old_parser/comment.go +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Cadence - The resource-oriented smart contract programming language - * - * Copyright Flow Foundation - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package old_parser - -// TODO(merge): Do we need this file? -import ( - "github.com/onflow/cadence/parser/lexer" -) - -func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { - var depth int - - for { - switch p.current.Type { - case lexer.TokenBlockCommentStart: - p.next() - depth++ - - case lexer.TokenBlockCommentContent: - p.next() - - case lexer.TokenBlockCommentEnd: - endToken = p.current - // Skip the comment end (`*/`) - p.next() - ok = true - depth-- - if depth == 0 { - return - } - - case lexer.TokenEOF: - p.reportSyntaxError( - "missing comment end %s", - lexer.TokenBlockCommentEnd, - ) - ok = false - return - - default: - p.reportSyntaxError( - "unexpected token %s in block comment", - p.current.Type, - ) - ok = false - return - } - } -} diff --git a/old_parser/declaration.go b/old_parser/declaration.go index 4fd075a8a9..ecda394dd7 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -1052,8 +1052,8 @@ func parseCompositeOrInterfaceDeclaration( identifier, []*ast.NominalType{}, members, - docString, declarationRange, + ast.Comments{}, ), nil } else { return ast.NewCompositeDeclaration( @@ -1155,8 +1155,8 @@ func parseAttachmentDeclaration( baseNominalType, conformances, members, - docString, declarationRange, + ast.Comments{}, ), nil } From 04126491178066c20e71ddb6af4066ce7555e58d Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 11 May 2025 17:39:53 +0200 Subject: [PATCH 40/84] clarify legacy field usage --- parser/lexer/state.go | 2 ++ parser/lexer/tokentype.go | 2 ++ tools/compatibility-check/go.mod | 2 +- 3 files changed, 5 insertions(+), 1 deletion(-) diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 2298287512..97a6dd449e 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -277,6 +277,8 @@ func numberState(l *lexer) stateFn { return rootState } +// Space must be preserved alongside the new comment structs, +// since some parsing code depends on it. type Space struct { ContainsNewline bool } diff --git a/parser/lexer/tokentype.go b/parser/lexer/tokentype.go index 2fd38270a8..c5e18d0476 100644 --- a/parser/lexer/tokentype.go +++ b/parser/lexer/tokentype.go @@ -29,6 +29,8 @@ const EOF rune = -1 const ( TokenError TokenType = iota TokenEOF + // TokenSpace must be preserved alongside the new comment structs, + // since some parsing code depends on it. TokenSpace TokenBinaryIntegerLiteral TokenOctalIntegerLiteral diff --git a/tools/compatibility-check/go.mod b/tools/compatibility-check/go.mod index e4755a3d03..34be713f08 100644 --- a/tools/compatibility-check/go.mod +++ b/tools/compatibility-check/go.mod @@ -1,6 +1,6 @@ module github.com/onflow/cadence/tools/compatibility_check -go 1.23 +go 1.23.0 require ( github.com/onflow/cadence v1.1.1-0.20241018202510-7f1b6fbc57c2 From 1cf6bfd5fa7a954a8a547e052b3c1b2ea9c630e6 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 11 May 2025 18:55:12 +0200 Subject: [PATCH 41/84] fix parser test issues --- ast/block.go | 1 + parser/declaration_test.go | 8 +++++--- parser/expression_test.go | 16 ++++++++++++++++ parser/parser_test.go | 25 ++++++++++++------------- 4 files changed, 34 insertions(+), 16 deletions(-) diff --git a/ast/block.go b/ast/block.go index 24bb38b8d4..3bbc68f172 100644 --- a/ast/block.go +++ b/ast/block.go @@ -40,6 +40,7 @@ func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange R return &Block{ Statements: statements, Range: astRange, + Comments: comments, } } diff --git a/parser/declaration_test.go b/parser/declaration_test.go index d38393805a..1144bb7a96 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -2762,9 +2762,6 @@ event E() // After E Leading: []*ast.Comment{ ast.NewComment(nil, []byte("// Before E")), }, - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// After E")), - }, }, Members: ast.NewUnmeteredMembers( []ast.Declaration{ @@ -2777,6 +2774,11 @@ event E() // After E StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, EndPos: ast.Position{Offset: 21, Line: 3, Column: 8}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After E")), + }, + }, }, StartPos: ast.Position{Offset: 20, Line: 3, Column: 7}, }, diff --git a/parser/expression_test.go b/parser/expression_test.go index 4200603531..d2bf651649 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -2160,6 +2160,11 @@ func TestParseBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/*test foo*/")), + }, + }, }, Right: &ast.IntegerExpression{ PositiveLiteral: []byte("2"), @@ -2169,6 +2174,11 @@ func TestParseBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 27, Offset: 27}, EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* bar */")), + }, + }, }, }, result, @@ -3013,6 +3023,12 @@ func TestParseLineComment(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 1, Offset: 28}, EndPos: ast.Position{Line: 2, Column: 1, Offset: 28}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("//// // this is a comment")), + }, + Trailing: nil, + }, }, Right: &ast.IntegerExpression{ PositiveLiteral: []byte("2"), diff --git a/parser/parser_test.go b/parser/parser_test.go index 23947d45e2..9d4f830782 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1012,20 +1012,13 @@ func TestParseComments(t *testing.T) { t.Parallel() - t.Run("special function declaration", func(t *testing.T) { + t.Run("event declaration", func(t *testing.T) { res, errs := ParseDeclarations( nil, []byte(` /// Before MyEvent event MyEvent() // After MyEvent -/// Before prepare -prepare() {} // After prepare -/// Before pre -pre {} // After pre -/// Before execute -execute {} // After execute -/// Before post -post {} // After post +/// Ignored `), Config{}, ) @@ -1033,13 +1026,19 @@ post {} // After post assert.Empty(t, errs) assert.NotNil(t, res) - event, ok := res[0].(*ast.SpecialFunctionDeclaration) + event, ok := res[0].(*ast.CompositeDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, + }, event.Comments) + assert.Equal(t, " Before MyEvent", event.DeclarationDocString()) + + decl, ok := event.Members.Declarations()[0].(*ast.SpecialFunctionDeclaration) assert.True(t, ok) assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, - }, event.FunctionDeclaration.Comments) - assert.Equal(t, " Before MyEvent", event.FunctionDeclaration.DeclarationDocString()) + }, decl.FunctionDeclaration.ParameterList.Comments) + assert.Equal(t, "", decl.DeclarationDocString()) }) From a9d992429cd5027c7547282202398b189f5abbcc Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 11 May 2025 19:18:26 +0200 Subject: [PATCH 42/84] add todo --- parser/expression_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/parser/expression_test.go b/parser/expression_test.go index d2bf651649..442f73ef2e 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -2146,6 +2146,8 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() + // TODO(preserve-comments): Extracting comments from operator tokens is a bit difficult, + // so let's handle this later as it seems a pretty edge case location to add comments. result, errs := testParseExpression(" 1/*test foo*/+/* bar */ 2 ") require.Empty(t, errs) From 1471e54a36604c6cdc7c2c53c55267355f6c7cd0 Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 12 May 2025 12:29:12 +0200 Subject: [PATCH 43/84] update if statement parsing --- ast/statement.go | 4 ++ parser/statement.go | 10 +++- parser/statement_test.go | 102 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 114 insertions(+), 2 deletions(-) diff --git a/ast/statement.go b/ast/statement.go index c211553f39..0a6cb179bb 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -209,6 +209,8 @@ type IfStatement struct { Then *Block Else *Block StartPos Position `json:"-"` + // Comments.Leading comments that appear before `if` keyword + Comments Comments } var _ Element = &IfStatement{} @@ -220,6 +222,7 @@ func NewIfStatement( thenBlock *Block, elseBlock *Block, startPos Position, + comments Comments, ) *IfStatement { common.UseMemory(gauge, common.IfStatementMemoryUsage) return &IfStatement{ @@ -227,6 +230,7 @@ func NewIfStatement( Then: thenBlock, Else: elseBlock, StartPos: startPos, + Comments: comments, } } diff --git a/parser/statement.go b/parser/statement.go index ab76510f64..51b0d91c02 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -308,7 +308,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { var ifStatements []*ast.IfStatement for { - startPos := p.current.StartPos + startToken := p.current p.nextSemanticToken() var variableDeclaration *ast.VariableDeclaration @@ -345,11 +345,14 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.skipSpace() if p.isToken(p.current, lexer.TokenIdentifier, KeywordElse) { + elseToken := p.current p.nextSemanticToken() + p.current.Comments.Leading = elseToken.Comments.PackToList() if p.isToken(p.current, lexer.TokenIdentifier, KeywordIf) { parseNested = true } else { elseBlock, err = parseBlock(p) + // TODO(preserve-comments): Update test case with line comments before last else if err != nil { return nil, err } @@ -371,7 +374,10 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { test, thenBlock, elseBlock, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + }, ) if variableDeclaration != nil { diff --git a/parser/statement_test.go b/parser/statement_test.go index f0071e6f6f..2335941482 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -677,6 +677,108 @@ func TestParseIfStatement(t *testing.T) { ) }) + t.Run("empty then, if-else, else, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// before if +if true { + // noop +} // after if +// before else-if +else if true { + // noop +} /* after else-if */ else { + // noop +} // after else +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, + EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, + EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after if")), + }, + }, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, + EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, + EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("/* after else-if */")), + }, + }, + }, + Else: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 9, Column: 27, Offset: 116}, + EndPos: ast.Position{Line: 11, Column: 0, Offset: 127}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after else")), + }, + }, + }, + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + EndPos: ast.Position{Line: 11, Column: 0, Offset: 127}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before if")), + }, + }, + }, + }, + result, + ) + }) + } func TestParseWhileStatement(t *testing.T) { From 27b320af9a8f465d5ae48991716b2e475ac11dad Mon Sep 17 00:00:00 2001 From: Bart Date: Mon, 12 May 2025 12:49:29 +0200 Subject: [PATCH 44/84] update if comments test case --- parser/statement.go | 9 +++- parser/statement_test.go | 98 ++++++++++++++++++++-------------------- 2 files changed, 57 insertions(+), 50 deletions(-) diff --git a/parser/statement.go b/parser/statement.go index 51b0d91c02..aac59c3bda 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -347,12 +347,17 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { if p.isToken(p.current, lexer.TokenIdentifier, KeywordElse) { elseToken := p.current p.nextSemanticToken() - p.current.Comments.Leading = elseToken.Comments.PackToList() + + // the parser ignores the `else` token, + // so the simplest solution is to attach the comments to the (next) `if` token + leadingComments := elseToken.Comments.PackToList() + leadingComments = append(leadingComments, p.current.Comments.Leading...) + p.current.Comments.Leading = leadingComments + if p.isToken(p.current, lexer.TokenIdentifier, KeywordIf) { parseNested = true } else { elseBlock, err = parseBlock(p) - // TODO(preserve-comments): Update test case with line comments before last else if err != nil { return nil, err } diff --git a/parser/statement_test.go b/parser/statement_test.go index 2335941482..16f0c4419e 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -687,6 +687,10 @@ if true { // noop } // after if // before else-if +else if true { + // noop +} +// before second else-if else if true { // noop } /* after else-if */ else { @@ -737,22 +741,57 @@ else if true { Comments: ast.Comments{ Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("/* after else-if */")), }, }, }, Else: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 9, Column: 27, Offset: 116}, - EndPos: ast.Position{Line: 11, Column: 0, Offset: 127}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("// after else")), + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, + EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, + EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("/* after else-if */")), + }, + }, + }, + Else: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after else")), + }, + }, + }, + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before second else-if")), + }, + }, }, }, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, }, StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, Comments: ast.Comments{ @@ -764,7 +803,7 @@ else if true { }, Range: ast.Range{ StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, - EndPos: ast.Position{Line: 11, Column: 0, Offset: 127}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, }, }, StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, @@ -778,43 +817,6 @@ else if true { result, ) }) - -} - -func TestParseWhileStatement(t *testing.T) { - - t.Parallel() - - t.Run("empty block", func(t *testing.T) { - - t.Parallel() - - result, errs := testParseStatements("while true { }") - require.Empty(t, errs) - - AssertEqualWithDiff(t, - []ast.Statement{ - &ast.WhileStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 6, Offset: 6}, - EndPos: ast.Position{Line: 1, Column: 9, Offset: 9}, - }, - }, - Block: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 11, Offset: 11}, - EndPos: ast.Position{Line: 1, Column: 13, Offset: 13}, - }, - }, - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - }, - }, - result, - ) - }) } func TestParseAssignmentStatement(t *testing.T) { From bb4952f1d53a87f0446920cb1fff4d85f3bf1ff3 Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 14 May 2025 12:04:39 +0200 Subject: [PATCH 45/84] update return, continue, break parsing --- ast/expression.go | 1 + ast/statement.go | 12 ++++-- old_parser/statement.go | 5 ++- parser/statement.go | 25 +++++++----- parser/statement_test.go | 83 ++++++++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+), 16 deletions(-) diff --git a/ast/expression.go b/ast/expression.go index de1d4a3302..313608020e 100644 --- a/ast/expression.go +++ b/ast/expression.go @@ -47,6 +47,7 @@ type Expression interface { type BoolExpression struct { Value bool Range + Comments } var _ Element = &BoolExpression{} diff --git a/ast/statement.go b/ast/statement.go index 0a6cb179bb..9a7ec0dcbd 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -104,15 +104,17 @@ func (s *ReturnStatement) MarshalJSON() ([]byte, error) { type BreakStatement struct { Range + Comments } var _ Element = &BreakStatement{} var _ Statement = &BreakStatement{} -func NewBreakStatement(gauge common.MemoryGauge, tokenRange Range) *BreakStatement { +func NewBreakStatement(gauge common.MemoryGauge, tokenRange Range, comments Comments) *BreakStatement { common.UseMemory(gauge, common.BreakStatementMemoryUsage) return &BreakStatement{ - Range: tokenRange, + Range: tokenRange, + Comments: comments, } } @@ -151,15 +153,17 @@ func (s *BreakStatement) MarshalJSON() ([]byte, error) { type ContinueStatement struct { Range + Comments } var _ Element = &ContinueStatement{} var _ Statement = &ContinueStatement{} -func NewContinueStatement(gauge common.MemoryGauge, tokenRange Range) *ContinueStatement { +func NewContinueStatement(gauge common.MemoryGauge, tokenRange Range, comments Comments) *ContinueStatement { common.UseMemory(gauge, common.ContinueStatementMemoryUsage) return &ContinueStatement{ - Range: tokenRange, + Range: tokenRange, + Comments: comments, } } diff --git a/old_parser/statement.go b/old_parser/statement.go index 6651fb6f74..079349f255 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -258,14 +258,14 @@ func parseBreakStatement(p *parser) *ast.BreakStatement { tokenRange := p.current.Range p.next() - return ast.NewBreakStatement(p.memoryGauge, tokenRange) + return ast.NewBreakStatement(p.memoryGauge, tokenRange, ast.Comments{}) } func parseContinueStatement(p *parser) *ast.ContinueStatement { tokenRange := p.current.Range p.next() - return ast.NewContinueStatement(p.memoryGauge, tokenRange) + return ast.NewContinueStatement(p.memoryGauge, tokenRange, ast.Comments{}) } func parseIfStatement(p *parser) (*ast.IfStatement, error) { @@ -337,6 +337,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { thenBlock, elseBlock, startPos, + ast.Comments{}, ) if variableDeclaration != nil { diff --git a/parser/statement.go b/parser/statement.go index aac59c3bda..235a4edf28 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -247,7 +247,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( } func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { - var endToken lexer.Token + var endToken *lexer.Token startToken := p.current tokenRange := startToken.Range endPosition := tokenRange.EndPos @@ -261,7 +261,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { var err error switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: - endToken = p.current + endToken = &p.current break default: if !sawNewLine { @@ -274,6 +274,14 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { } } + comments := ast.Comments{} + if endToken == nil { + comments = startToken.Comments + } else { + comments.Leading = startToken.Comments.PackToList() + comments.Trailing = endToken.Comments.PackToList() + } + return ast.NewReturnStatement( p.memoryGauge, expression, @@ -282,25 +290,22 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), - ast.Comments{ - Leading: startToken.Comments.PackToList(), - Trailing: endToken.Comments.PackToList(), - }, + comments, ), nil } func parseBreakStatement(p *parser) *ast.BreakStatement { - tokenRange := p.current.Range + breakToken := p.current p.next() - return ast.NewBreakStatement(p.memoryGauge, tokenRange) + return ast.NewBreakStatement(p.memoryGauge, breakToken.Range, breakToken.Comments) } func parseContinueStatement(p *parser) *ast.ContinueStatement { - tokenRange := p.current.Range + continueToken := p.current p.next() - return ast.NewContinueStatement(p.memoryGauge, tokenRange) + return ast.NewContinueStatement(p.memoryGauge, continueToken.Range, continueToken.Comments) } func parseIfStatement(p *parser) (*ast.IfStatement, error) { diff --git a/parser/statement_test.go b/parser/statement_test.go index 16f0c4419e..4b39ddd39f 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -2124,6 +2124,89 @@ func TestParseWhileStatementInFunctionDeclaration(t *testing.T) { ) } +func TestParseReturnBreakContinueWithComments(t *testing.T) { + + t.Parallel() + + const code = ` + // Before return + return // After return + // Before return semicolon + return /* After return, before semicolon */ ; // After semicolon + // Before break + break // After break + // Before continue + continue // After continue + ` + result, errs := testParseStatements(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 31, Line: 3, Column: 8}, + EndPos: ast.Position{Offset: 36, Line: 3, Column: 13}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After return")), + }, + }, + }, + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 91, Line: 5, Column: 8}, + EndPos: ast.Position{Offset: 96, Line: 5, Column: 13}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before return semicolon")), + ast.NewComment(nil, []byte("/* After return, before semicolon */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After semicolon")), + }, + }, + }, + &ast.BreakStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 182, Line: 7, Column: 8}, + EndPos: ast.Position{Offset: 186, Line: 7, Column: 12}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before break")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After break")), + }, + }, + }, + &ast.ContinueStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 232, Line: 9, Column: 8}, + EndPos: ast.Position{Offset: 239, Line: 9, Column: 15}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before continue")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After continue")), + }, + }, + }, + }, + result, + ) +} + func TestParseForStatementInFunctionDeclaration(t *testing.T) { t.Parallel() From 1c7e1a7877126c474004fd9f697417a656205caa Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 14 May 2025 12:49:10 +0200 Subject: [PATCH 46/84] update switch/case parsing --- ast/statement.go | 19 ++++++ old_parser/statement.go | 1 + parser/statement.go | 31 ++++++---- parser/statement_test.go | 126 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 166 insertions(+), 11 deletions(-) diff --git a/ast/statement.go b/ast/statement.go index 9a7ec0dcbd..0125cdb169 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -761,6 +761,7 @@ type SwitchStatement struct { Expression Expression Cases []*SwitchCase Range + Comments } var _ Element = &SwitchStatement{} @@ -771,12 +772,14 @@ func NewSwitchStatement( expression Expression, cases []*SwitchCase, stmtRange Range, + comments Comments, ) *SwitchStatement { common.UseMemory(gauge, common.SwitchStatementMemoryUsage) return &SwitchStatement{ Expression: expression, Cases: cases, Range: stmtRange, + Comments: comments, } } @@ -855,6 +858,22 @@ type SwitchCase struct { Expression Expression Statements []Statement Range + Comments +} + +func NewSwitchCase( + _ common.MemoryGauge, + expression Expression, + statements []Statement, + r Range, + comments Comments, +) *SwitchCase { + return &SwitchCase{ + Expression: expression, + Statements: statements, + Range: r, + Comments: comments, + } } func (s *SwitchCase) MarshalJSON() ([]byte, error) { diff --git a/old_parser/statement.go b/old_parser/statement.go index 079349f255..795f4bf32d 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -656,6 +656,7 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { startPos, endToken.EndPos, ), + ast.Comments{}, ), nil } diff --git a/parser/statement.go b/parser/statement.go index 235a4edf28..d273bc853a 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -703,7 +703,7 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the `switch` keyword p.next() @@ -734,9 +734,13 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { cases, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ), + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: endToken.Comments.PackToList(), + }, ), nil } @@ -796,7 +800,7 @@ func parseSwitchCases(p *parser) (cases []*ast.SwitchCase, err error) { // | `default` `:` statements func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the keyword p.next() @@ -813,7 +817,7 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { p.skipSpace() } - colonPos := p.current.StartPos + colonToken := p.current if !p.current.Is(lexer.TokenColon) { p.reportSyntaxError( @@ -846,22 +850,27 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { return nil, err } - endPos := colonPos + endPos := colonToken.StartPos if len(statements) > 0 { lastStatementIndex := len(statements) - 1 endPos = statements[lastStatementIndex].EndPosition(p.memoryGauge) } - return &ast.SwitchCase{ - Expression: expression, - Statements: statements, - Range: ast.NewRange( + return ast.NewSwitchCase( + p.memoryGauge, + expression, + statements, + ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endPos, ), - }, nil + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: colonToken.Comments.PackToList(), + }, + ), nil } func parseRemoveStatement( diff --git a/parser/statement_test.go b/parser/statement_test.go index 4b39ddd39f..db2aa2cd03 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -1707,6 +1707,132 @@ func TestParseSwitchStatement(t *testing.T) { ) }) + t.Run("two cases, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` + // Before switch + switch x { + // Before first case + case true : // After first case + true + // Before default case + default : /* After default case */ false + } // After switch +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.SwitchStatement{ + Expression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 10, Offset: 31}, + }, + }, + Cases: []*ast.SwitchCase{ + { + Expression: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 70, + Line: 5, + Column: 9, + }, + EndPos: ast.Position{ + Offset: 73, + Line: 5, + Column: 12, + }, + }, + }, + Statements: []ast.Statement{ + &ast.ExpressionStatement{ + Expression: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 102, + Line: 6, + Column: 5, + }, + EndPos: ast.Position{ + Offset: 105, + Line: 6, + Column: 8, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 5, Column: 4, Offset: 65}, + EndPos: ast.Position{Line: 6, Column: 8, Offset: 105}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before first case")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After first case")), + }, + }, + }, + { + Statements: []ast.Statement{ + &ast.ExpressionStatement{ + Expression: &ast.BoolExpression{ + Value: false, + Range: ast.Range{ + StartPos: ast.Position{ + Offset: 173, + Line: 8, + Column: 39, + }, + EndPos: ast.Position{ + Offset: 177, + Line: 8, + Column: 43, + }, + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 8, Column: 4, Offset: 138}, + EndPos: ast.Position{Line: 8, Column: 43, Offset: 177}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before default case")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* After default case */")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 24}, + EndPos: ast.Position{Line: 9, Column: 3, Offset: 182}, + }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before switch")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// After switch")), + }, + }, + }, + }, + result, + ) + }) + t.Run("Invalid identifiers in switch cases", func(t *testing.T) { code := "switch 1 {AAAAA: break; case 3: break; default: break}" _, errs := testParseStatements(code) From ef4fab58d614d83e2d046678ef97438237928fe9 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 15 May 2025 11:42:46 +0200 Subject: [PATCH 47/84] update while parsing --- ast/statement.go | 3 + old_parser/statement.go | 2 +- parser/statement.go | 12 ++- parser/statement_test.go | 158 +++++++++++++++++++++++++-------------- 4 files changed, 117 insertions(+), 58 deletions(-) diff --git a/ast/statement.go b/ast/statement.go index 0125cdb169..9250c32bca 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -324,6 +324,7 @@ type WhileStatement struct { Test Expression Block *Block StartPos Position `json:"-"` + Comments } var _ Element = &WhileStatement{} @@ -334,12 +335,14 @@ func NewWhileStatement( expression Expression, block *Block, startPos Position, + comments Comments, ) *WhileStatement { common.UseMemory(gauge, common.WhileStatementMemoryUsage) return &WhileStatement{ Test: expression, Block: block, StartPos: startPos, + Comments: comments, } } diff --git a/old_parser/statement.go b/old_parser/statement.go index 795f4bf32d..1ee53954b7 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -384,7 +384,7 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { return nil, err } - return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos), nil + return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos, ast.Comments{}), nil } func parseForStatement(p *parser) (*ast.ForStatement, error) { diff --git a/parser/statement.go b/parser/statement.go index d273bc853a..fcadc43428 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -421,7 +421,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() expression, err := parseExpression(p, lowestBindingPower) @@ -434,7 +434,15 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { return nil, err } - return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos), nil + return ast.NewWhileStatement( + p.memoryGauge, + expression, + block, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + }, + ), nil } func parseForStatement(p *parser) (*ast.ForStatement, error) { diff --git a/parser/statement_test.go b/parser/statement_test.go index db2aa2cd03..4f0a5705f3 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -2166,11 +2166,12 @@ func TestParseIfStatementNoElse(t *testing.T) { ) } -func TestParseWhileStatementInFunctionDeclaration(t *testing.T) { +func TestParseWhileStatement(t *testing.T) { t.Parallel() - const code = ` + t.Run("while in function, comments", func(t *testing.T) { + const code = ` fun test() { while true { return @@ -2179,75 +2180,122 @@ func TestParseWhileStatementInFunctionDeclaration(t *testing.T) { } } ` - result, errs := testParseProgram(code) - require.Empty(t, errs) + result, errs := testParseProgram(code) + require.Empty(t, errs) - AssertEqualWithDiff(t, - []ast.Declaration{ - &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Identifier: ast.Identifier{ - Identifier: "test", - Pos: ast.Position{Offset: 10, Line: 2, Column: 9}, - }, - ParameterList: &ast.ParameterList{ - Range: ast.Range{ - StartPos: ast.Position{Offset: 14, Line: 2, Column: 13}, - EndPos: ast.Position{Offset: 15, Line: 2, Column: 14}, + AssertEqualWithDiff(t, + []ast.Declaration{ + &ast.FunctionDeclaration{ + Access: ast.AccessNotSpecified, + Identifier: ast.Identifier{ + Identifier: "test", + Pos: ast.Position{Offset: 10, Line: 2, Column: 9}, }, - }, - FunctionBlock: &ast.FunctionBlock{ - Block: &ast.Block{ - Statements: []ast.Statement{ - &ast.WhileStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Offset: 37, Line: 3, Column: 18}, - EndPos: ast.Position{Offset: 40, Line: 3, Column: 21}, + ParameterList: &ast.ParameterList{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 14, Line: 2, Column: 13}, + EndPos: ast.Position{Offset: 15, Line: 2, Column: 14}, + }, + }, + FunctionBlock: &ast.FunctionBlock{ + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.WhileStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Offset: 37, Line: 3, Column: 18}, + EndPos: ast.Position{Offset: 40, Line: 3, Column: 21}, + }, }, - }, - Block: &ast.Block{ - Statements: []ast.Statement{ - &ast.ReturnStatement{ - Expression: nil, - Range: ast.Range{ - StartPos: ast.Position{Offset: 58, Line: 4, Column: 14}, - EndPos: ast.Position{Offset: 63, Line: 4, Column: 19}, + Block: &ast.Block{ + Statements: []ast.Statement{ + &ast.ReturnStatement{ + Expression: nil, + Range: ast.Range{ + StartPos: ast.Position{Offset: 58, Line: 4, Column: 14}, + EndPos: ast.Position{Offset: 63, Line: 4, Column: 19}, + }, }, - }, - &ast.BreakStatement{ - Range: ast.Range{ - StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, - EndPos: ast.Position{Offset: 83, Line: 5, Column: 18}, + &ast.BreakStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 79, Line: 5, Column: 14}, + EndPos: ast.Position{Offset: 83, Line: 5, Column: 18}, + }, }, - }, - &ast.ContinueStatement{ - Range: ast.Range{ - StartPos: ast.Position{Offset: 99, Line: 6, Column: 14}, - EndPos: ast.Position{Offset: 106, Line: 6, Column: 21}, + &ast.ContinueStatement{ + Range: ast.Range{ + StartPos: ast.Position{Offset: 99, Line: 6, Column: 14}, + EndPos: ast.Position{Offset: 106, Line: 6, Column: 21}, + }, }, }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 42, Line: 3, Column: 23}, + EndPos: ast.Position{Offset: 120, Line: 7, Column: 12}, + }, }, - Range: ast.Range{ - StartPos: ast.Position{Offset: 42, Line: 3, Column: 23}, - EndPos: ast.Position{Offset: 120, Line: 7, Column: 12}, - }, + StartPos: ast.Position{Offset: 31, Line: 3, Column: 12}, }, - StartPos: ast.Position{Offset: 31, Line: 3, Column: 12}, }, + Range: ast.Range{ + StartPos: ast.Position{Offset: 17, Line: 2, Column: 16}, + EndPos: ast.Position{Offset: 130, Line: 8, Column: 8}, + }, + }, + }, + StartPos: ast.Position{Offset: 6, Line: 2, Column: 5}, + }, + }, + result.Declarations(), + ) + }) + + t.Run("while, comments", func(t *testing.T) { + const code = ` + // before while + while true { + // ignore + } // after while + ` + result, errs := testParseStatements(code) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.WhileStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Offset: 30, Line: 3, Column: 11}, + EndPos: ast.Position{Offset: 33, Line: 3, Column: 14}, }, + }, + Block: &ast.Block{ + Statements: []ast.Statement{}, Range: ast.Range{ - StartPos: ast.Position{Offset: 17, Line: 2, Column: 16}, - EndPos: ast.Position{Offset: 130, Line: 8, Column: 8}, + StartPos: ast.Position{Offset: 35, Line: 3, Column: 16}, + EndPos: ast.Position{Offset: 53, Line: 5, Column: 2}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// ignore")), + ast.NewComment(nil, []byte("// after while")), + }, + }, + }, + StartPos: ast.Position{Offset: 24, Line: 3, Column: 5}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before while")), }, }, }, - StartPos: ast.Position{Offset: 6, Line: 2, Column: 5}, }, - }, - result.Declarations(), - ) + result, + ) + }) + } func TestParseReturnBreakContinueWithComments(t *testing.T) { From 6a956b0d758bec9083507a8838a6d008767b88b5 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 15 May 2025 12:15:38 +0200 Subject: [PATCH 48/84] update remove from parsing --- ast/attachment.go | 3 +++ old_parser/statement.go | 1 + parser/statement.go | 14 +++++++++--- parser/statement_test.go | 46 +++++++++++++++++++++++++++++++++++++--- 4 files changed, 58 insertions(+), 6 deletions(-) diff --git a/ast/attachment.go b/ast/attachment.go index 702780adf0..cef2157ac8 100644 --- a/ast/attachment.go +++ b/ast/attachment.go @@ -298,6 +298,7 @@ type RemoveStatement struct { Attachment *NominalType Value Expression StartPos Position `json:"-"` + Comments } var _ Element = &RemoveStatement{} @@ -308,6 +309,7 @@ func NewRemoveStatement( attachment *NominalType, value Expression, startPos Position, + comments Comments, ) *RemoveStatement { common.UseMemory(gauge, common.RemoveStatementMemoryUsage) @@ -315,6 +317,7 @@ func NewRemoveStatement( Attachment: attachment, Value: value, StartPos: startPos, + Comments: comments, } } diff --git a/old_parser/statement.go b/old_parser/statement.go index 1ee53954b7..d5f5af6d75 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -826,5 +826,6 @@ func parseRemoveStatement( attachmentNominalType, attached, startPos, + ast.Comments{}, ), nil } diff --git a/parser/statement.go b/parser/statement.go index fcadc43428..8559ae086c 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -885,7 +885,7 @@ func parseRemoveStatement( p *parser, ) (*ast.RemoveStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() p.skipSpace() @@ -904,8 +904,11 @@ func parseRemoveStatement( p.skipSpace() + var fromToken lexer.Token // check and skip `from` keyword - if !p.isToken(p.current, lexer.TokenIdentifier, KeywordFrom) { + if p.isToken(p.current, lexer.TokenIdentifier, KeywordFrom) { + fromToken = p.current + } else { p.reportSyntaxError( "expected from keyword, got %s", p.current.Type, @@ -918,10 +921,15 @@ func parseRemoveStatement( return nil, err } + leadingComments := startToken.Comments.PackToList() + leadingComments = append(leadingComments, fromToken.Comments.PackToList()...) return ast.NewRemoveStatement( p.memoryGauge, attachmentNominalType, attached, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: leadingComments, + }, ), nil } diff --git a/parser/statement_test.go b/parser/statement_test.go index 4f0a5705f3..977879b502 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -1450,16 +1450,56 @@ func TestParseRemoveAttachmentStatement(t *testing.T) { Attachment: &ast.NominalType{ Identifier: ast.Identifier{ Identifier: "A", - Pos: ast.Position{Line: 1, Column: 7, Offset: 7}, + Pos: ast.Position{Line: 3, Column: 7, Offset: 25}, }, }, Value: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "b", - Pos: ast.Position{Line: 1, Column: 14, Offset: 14}, + Pos: ast.Position{Line: 5, Column: 5, Offset: 47}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + }, + }, + result, + ) + }) + + t.Run("basic, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before remove +remove A +// Before from +from b +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.RemoveStatement{ + Attachment: &ast.NominalType{ + Identifier: ast.Identifier{ + Identifier: "A", + Pos: ast.Position{Line: 3, Column: 7, Offset: 25}, + }, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "b", + Pos: ast.Position{Line: 5, Column: 5, Offset: 47}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before remove")), + ast.NewComment(nil, []byte("// Before from")), }, }, - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, }, result, From 9486f3c1c13eacdd0f164146d8d7f68acdd9974a Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 15 May 2025 12:26:16 +0200 Subject: [PATCH 49/84] update emit statement parsing --- ast/statement.go | 3 +++ old_parser/statement.go | 2 +- parser/statement.go | 11 +++++++++-- parser/statement_test.go | 36 ++++++++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+), 3 deletions(-) diff --git a/ast/statement.go b/ast/statement.go index 9250c32bca..0fcdf64b7b 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -498,6 +498,7 @@ func (s *ForStatement) MarshalJSON() ([]byte, error) { type EmitStatement struct { InvocationExpression *InvocationExpression StartPos Position `json:"-"` + Comments } var _ Element = &EmitStatement{} @@ -507,11 +508,13 @@ func NewEmitStatement( gauge common.MemoryGauge, invocation *InvocationExpression, startPos Position, + comments Comments, ) *EmitStatement { common.UseMemory(gauge, common.EmitStatementMemoryUsage) return &EmitStatement{ InvocationExpression: invocation, StartPos: startPos, + Comments: comments, } } diff --git a/old_parser/statement.go b/old_parser/statement.go index d5f5af6d75..01d0d12996 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -617,7 +617,7 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { return nil, err } - return ast.NewEmitStatement(p.memoryGauge, invocation, startPos), nil + return ast.NewEmitStatement(p.memoryGauge, invocation, startPos, ast.Comments{}), nil } func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { diff --git a/parser/statement.go b/parser/statement.go index 8559ae086c..65a66856ad 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -698,7 +698,7 @@ func parseCondition(p *parser) (ast.Condition, error) { } func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.next() invocation, err := parseNominalTypeInvocationRemainder(p) @@ -706,7 +706,14 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { return nil, err } - return ast.NewEmitStatement(p.memoryGauge, invocation, startPos), nil + return ast.NewEmitStatement( + p.memoryGauge, + invocation, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + }, + ), nil } func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { diff --git a/parser/statement_test.go b/parser/statement_test.go index 977879b502..d762edb4f1 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -1149,6 +1149,42 @@ func TestParseEmit(t *testing.T) { result, ) }) + + t.Run("simple, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before emit 1 +/* Before emit 2 */ emit T() +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.EmitStatement{ + InvocationExpression: &ast.InvocationExpression{ + InvokedExpression: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "T", + Pos: ast.Position{Line: 3, Column: 25, Offset: 43}, + }, + }, + ArgumentsStartPos: ast.Position{Line: 3, Column: 26, Offset: 44}, + EndPos: ast.Position{Line: 3, Column: 27, Offset: 45}, + }, + StartPos: ast.Position{Line: 3, Column: 20, Offset: 38}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before emit 1")), + ast.NewComment(nil, []byte("/* Before emit 2 */")), + }, + }, + }, + }, + result, + ) + }) } func TestParseFunctionStatementOrExpression(t *testing.T) { From d8c32df53c01dc304521fbf5281b08ba106e4943 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 15 May 2025 13:07:26 +0200 Subject: [PATCH 50/84] update for statement parsing, start attaching comments to identifiers --- ast/identifier.go | 16 +++++++++++++ ast/statement.go | 3 +++ old_parser/statement.go | 1 + parser/parser.go | 3 ++- parser/statement.go | 13 +++++++--- parser/statement_test.go | 51 ++++++++++++++++++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 4 deletions(-) diff --git a/ast/identifier.go b/ast/identifier.go index 6acf9ebc46..f49f2ac1f9 100644 --- a/ast/identifier.go +++ b/ast/identifier.go @@ -29,6 +29,7 @@ import ( type Identifier struct { Identifier string Pos Position + Comments Comments } func NewIdentifier(memoryGauge common.MemoryGauge, identifier string, pos Position) Identifier { @@ -39,6 +40,21 @@ func NewIdentifier(memoryGauge common.MemoryGauge, identifier string, pos Positi } } +// NewIdentifierWithComments TODO(preserve-comments): Migrate to NewIdentifier? +func NewIdentifierWithComments( + memoryGauge common.MemoryGauge, + identifier string, + pos Position, + comments Comments, +) Identifier { + common.UseMemory(memoryGauge, common.IdentifierMemoryUsage) + return Identifier{ + Identifier: identifier, + Pos: pos, + Comments: comments, + } +} + func NewEmptyIdentifier(memoryGauge common.MemoryGauge, pos Position) Identifier { common.UseMemory(memoryGauge, common.IdentifierMemoryUsage) return Identifier{ diff --git a/ast/statement.go b/ast/statement.go index 0fcdf64b7b..fcbcbae582 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -403,6 +403,7 @@ type ForStatement struct { Block *Block Identifier Identifier StartPos Position `json:"-"` + Comments } var _ Element = &ForStatement{} @@ -415,6 +416,7 @@ func NewForStatement( block *Block, expression Expression, startPos Position, + comments Comments, ) *ForStatement { common.UseMemory(gauge, common.ForStatementMemoryUsage) @@ -424,6 +426,7 @@ func NewForStatement( Block: block, Value: expression, StartPos: startPos, + Comments: comments, } } diff --git a/old_parser/statement.go b/old_parser/statement.go index 01d0d12996..80535ae362 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -450,6 +450,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { block, expression, startPos, + ast.Comments{}, ), nil } diff --git a/parser/parser.go b/parser/parser.go index 3b60ee6551..a3cf9c1a74 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -506,10 +506,11 @@ func (p *parser) nonReservedIdentifier(errMsgContext string) (ast.Identifier, er } func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { - return ast.NewIdentifier( + return ast.NewIdentifierWithComments( p.memoryGauge, string(p.tokenSource(token)), token.StartPos, + token.Comments, ) } diff --git a/parser/statement.go b/parser/statement.go index 65a66856ad..0ff55a7abb 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -447,7 +447,7 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { func parseForStatement(p *parser) (*ast.ForStatement, error) { - startPos := p.current.StartPos + startToken := p.current p.nextSemanticToken() if p.isToken(p.current, lexer.TokenIdentifier, KeywordIn) { @@ -481,7 +481,10 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { identifier = firstValue } - if !p.isToken(p.current, lexer.TokenIdentifier, KeywordIn) { + var inToken lexer.Token + if p.isToken(p.current, lexer.TokenIdentifier, KeywordIn) { + inToken = p.current + } else { p.reportSyntaxError( "expected keyword %q, got %s", KeywordIn, @@ -507,7 +510,11 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { index, block, expression, - startPos, + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + Trailing: inToken.Comments.PackToList(), + }, ), nil } diff --git a/parser/statement_test.go b/parser/statement_test.go index d762edb4f1..cf2c8bf99c 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -1034,6 +1034,57 @@ func TestParseForStatement(t *testing.T) { result, ) }) + + t.Run("empty block, comments", func(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// Before for +for /* after for */ x /* before in */ in /* after in */ y { } +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.ForStatement{ + Identifier: ast.Identifier{ + Identifier: "x", + Pos: ast.Position{Line: 3, Column: 20, Offset: 35}, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* before in */")), + }, + }, + }, + Value: &ast.IdentifierExpression{ + Identifier: ast.Identifier{ + Identifier: "y", + Pos: ast.Position{Line: 3, Column: 56, Offset: 71}, + }, + }, + Block: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 58, Offset: 73}, + EndPos: ast.Position{Line: 3, Column: 60, Offset: 75}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 15}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// Before for")), + ast.NewComment(nil, []byte("/* after for */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/* after in */")), + }, + }, + }, + }, + result, + ) + }) } func TestParseForStatementIndexBinding(t *testing.T) { From b7feec6fb5e0d9918a5d2b55308930491b1784e7 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 10 Jun 2025 07:10:38 +0200 Subject: [PATCH 51/84] update function declaration or expression stmt parsing --- ast/function_declaration.go | 36 ++---------------------------------- ast/identifier.go | 7 +------ old_parser/declaration.go | 5 +++-- old_parser/function.go | 2 +- old_parser/statement.go | 10 ++++++---- old_parser/transaction.go | 2 +- parser/declaration.go | 4 ++-- parser/function.go | 2 +- parser/statement.go | 25 ++++++++++++++++++------- parser/transaction.go | 2 +- 10 files changed, 36 insertions(+), 59 deletions(-) diff --git a/ast/function_declaration.go b/ast/function_declaration.go index f9710eb4cc..c4bcc13744 100644 --- a/ast/function_declaration.go +++ b/ast/function_declaration.go @@ -79,39 +79,6 @@ var _ Element = &FunctionDeclaration{} var _ Declaration = &FunctionDeclaration{} var _ Statement = &FunctionDeclaration{} -// TODO(preserve-comments): Temporary, add `comments` param to NewFunctionDeclaration in the future -func NewFunctionDeclarationWithComments( - gauge common.MemoryGauge, - access Access, - purity FunctionPurity, - isStatic bool, - isNative bool, - identifier Identifier, - typeParameterList *TypeParameterList, - parameterList *ParameterList, - returnTypeAnnotation *TypeAnnotation, - functionBlock *FunctionBlock, - startPos Position, - comments Comments, -) *FunctionDeclaration { - decl := NewFunctionDeclaration( - gauge, - access, - purity, - isStatic, - isNative, - identifier, - typeParameterList, - parameterList, - returnTypeAnnotation, - functionBlock, - startPos, - "", - ) - decl.Comments = comments - return decl -} - func NewFunctionDeclaration( gauge common.MemoryGauge, access Access, @@ -124,7 +91,7 @@ func NewFunctionDeclaration( returnTypeAnnotation *TypeAnnotation, functionBlock *FunctionBlock, startPos Position, - docString string, + comments Comments, ) *FunctionDeclaration { common.UseMemory(gauge, common.FunctionDeclarationMemoryUsage) @@ -146,6 +113,7 @@ func NewFunctionDeclaration( ReturnTypeAnnotation: returnTypeAnnotation, FunctionBlock: functionBlock, StartPos: startPos, + Comments: comments, } } diff --git a/ast/identifier.go b/ast/identifier.go index f49f2ac1f9..e5e6bd7699 100644 --- a/ast/identifier.go +++ b/ast/identifier.go @@ -33,14 +33,9 @@ type Identifier struct { } func NewIdentifier(memoryGauge common.MemoryGauge, identifier string, pos Position) Identifier { - common.UseMemory(memoryGauge, common.IdentifierMemoryUsage) - return Identifier{ - Identifier: identifier, - Pos: pos, - } + return NewIdentifierWithComments(memoryGauge, identifier, pos, Comments{}) } -// NewIdentifierWithComments TODO(preserve-comments): Migrate to NewIdentifier? func NewIdentifierWithComments( memoryGauge common.MemoryGauge, identifier string, diff --git a/old_parser/declaration.go b/old_parser/declaration.go index ecda394dd7..95e59ee106 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -805,7 +805,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - "", + ast.Comments{}, ), ) @@ -1486,6 +1486,7 @@ func parseSpecialFunctionDeclaration( staticPos *ast.Position, nativePos *ast.Position, identifier ast.Identifier, + // Comments can be safely ignored docString string, ) (*ast.SpecialFunctionDeclaration, error) { @@ -1539,7 +1540,7 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - docString, + ast.Comments{}, ), ), nil } diff --git a/old_parser/function.go b/old_parser/function.go index 6fa1aa3a80..83c67b3699 100644 --- a/old_parser/function.go +++ b/old_parser/function.go @@ -331,7 +331,7 @@ func parseFunctionDeclaration( return nil, err } - return ast.NewFunctionDeclarationWithComments( + return ast.NewFunctionDeclaration( p.memoryGauge, access, ast.FunctionPurityUnspecified, diff --git a/old_parser/statement.go b/old_parser/statement.go index 80535ae362..7cbbc2bc0b 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -155,7 +155,7 @@ func parseStatement(p *parser) (ast.Statement, error) { func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.Statement, error) { - startPos := p.current.StartPos + startToken := p.current // Skip the `fun` keyword p.nextSemanticToken() @@ -193,8 +193,10 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State parameterList, returnTypeAnnotation, functionBlock, - startPos, - "", + startToken.StartPos, + ast.Comments{ + Leading: startToken.Comments.PackToList(), + }, ), nil } else { parameterList, returnTypeAnnotation, functionBlock, err := @@ -211,7 +213,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State parameterList, returnTypeAnnotation, functionBlock, - startPos, + startToken.StartPos, ), ), nil } diff --git a/old_parser/transaction.go b/old_parser/transaction.go index e920d7f621..2597a67e59 100644 --- a/old_parser/transaction.go +++ b/old_parser/transaction.go @@ -282,7 +282,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - "", + ast.Comments{}, ), ), nil } diff --git a/parser/declaration.go b/parser/declaration.go index f20da55fd4..3e9f06b71c 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -983,7 +983,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - "", + ast.Comments{}, ), ) @@ -1995,7 +1995,7 @@ func parseSpecialFunctionDeclaration( return ast.NewSpecialFunctionDeclaration( p.memoryGauge, declarationKind, - ast.NewFunctionDeclarationWithComments( + ast.NewFunctionDeclaration( p.memoryGauge, access, purity, diff --git a/parser/function.go b/parser/function.go index 2fd45a1d16..5fb5a60c3c 100644 --- a/parser/function.go +++ b/parser/function.go @@ -393,7 +393,7 @@ func parseFunctionDeclaration( leadingComments = append(leadingComments, startComments...) leadingComments = append(leadingComments, startToken.Comments.Leading...) - return ast.NewFunctionDeclarationWithComments( + return ast.NewFunctionDeclaration( p.memoryGauge, access, purity, diff --git a/parser/statement.go b/parser/statement.go index 0ff55a7abb..e14184cfe5 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -103,17 +103,16 @@ func parseStatement(p *parser) (ast.Statement, error) { case KeywordView: // save current stream state before looking ahead for the `fun` keyword cursor := p.tokens.Cursor() - current := p.current - purityPos := current.StartPos + purityToken := p.current p.nextSemanticToken() if p.isToken(p.current, lexer.TokenIdentifier, KeywordFun) { - return parseFunctionDeclarationOrFunctionExpressionStatement(p, ast.FunctionPurityView, &purityPos) + return parseFunctionDeclarationOrFunctionExpressionStatement(p, ast.FunctionPurityView, &purityToken) } // no `fun` :( revert back to previous lexer state and treat it as an identifier p.tokens.Revert(cursor) - p.current = current + p.current = purityToken tokenIsIdentifier = true case KeywordFun: @@ -178,10 +177,20 @@ func parseStatement(p *parser) (ast.Statement, error) { func parseFunctionDeclarationOrFunctionExpressionStatement( p *parser, purity ast.FunctionPurity, - purityPos *ast.Position, + purityToken *lexer.Token, ) (ast.Statement, error) { + var leadingComments []*ast.Comment + var startPos ast.Position - startPos := *ast.EarlierPosition(&p.current.StartPos, purityPos) + funToken := p.current + if purityToken == nil { + startPos = funToken.StartPos + } else { + startPos = *ast.EarlierPosition(&funToken.StartPos, &purityToken.StartPos) + leadingComments = append(leadingComments, purityToken.Comments.PackToList()...) + } + + leadingComments = append(leadingComments, funToken.Comments.PackToList()...) // Skip the `fun` keyword p.nextSemanticToken() @@ -223,7 +232,9 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( returnTypeAnnotation, functionBlock, startPos, - "", + ast.Comments{ + Leading: leadingComments, + }, ), nil } else { parameterList, returnTypeAnnotation, functionBlock, err := diff --git a/parser/transaction.go b/parser/transaction.go index 9d2ccc1cff..b8dc99b87f 100644 --- a/parser/transaction.go +++ b/parser/transaction.go @@ -275,7 +275,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) return ast.NewSpecialFunctionDeclaration( p.memoryGauge, common.DeclarationKindExecute, - ast.NewFunctionDeclarationWithComments( + ast.NewFunctionDeclaration( p.memoryGauge, ast.AccessNotSpecified, ast.FunctionPurityUnspecified, From db9f2168478a3fca4f71012eac2548eb1add8fa4 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 10 Jun 2025 20:51:44 +0200 Subject: [PATCH 52/84] fix lexer tests --- ast/identifier.go | 9 +-- parser/lexer/lexer.go | 11 +++- parser/lexer/lexer_test.go | 113 ++++++++++++++++++++++++++++++------- parser/lexer/state.go | 4 +- parser/parser.go | 4 +- 5 files changed, 107 insertions(+), 34 deletions(-) diff --git a/ast/identifier.go b/ast/identifier.go index e5e6bd7699..dc545498d6 100644 --- a/ast/identifier.go +++ b/ast/identifier.go @@ -29,24 +29,17 @@ import ( type Identifier struct { Identifier string Pos Position - Comments Comments } -func NewIdentifier(memoryGauge common.MemoryGauge, identifier string, pos Position) Identifier { - return NewIdentifierWithComments(memoryGauge, identifier, pos, Comments{}) -} - -func NewIdentifierWithComments( +func NewIdentifier( memoryGauge common.MemoryGauge, identifier string, pos Position, - comments Comments, ) Identifier { common.UseMemory(memoryGauge, common.IdentifierMemoryUsage) return Identifier{ Identifier: identifier, Pos: pos, - Comments: comments, } } diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index 8fe09bd5eb..d9c0a12a32 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -300,7 +300,7 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co Comments: ast.Comments{ Leading: leadingComments, // Trailing comments can't be determined, as it wasn't consumed yet at this point. - Trailing: []*ast.Comment{}, + Trailing: nil, }, } @@ -347,6 +347,15 @@ func (l *lexer) updatePreviousTrailingComments() { if lastNonSpaceTokenIndex != -1 { lastNonSpaceToken := &l.tokens[lastNonSpaceTokenIndex] + + if len(trailing) == 0 { + return + } + + if lastNonSpaceToken.Trailing == nil { + lastNonSpaceToken.Trailing = []*ast.Comment{} + } + lastNonSpaceToken.Trailing = append(lastNonSpaceToken.Trailing, trailing...) } } diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 20aa5bd1e4..daa574e96b 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -877,10 +877,32 @@ func TestLexBasic(t *testing.T) { ) }) - t.Run("Trivia", func(t *testing.T) { + t.Run("trivia", func(t *testing.T) { testLex(t, "// test is in next line \n/* test is here */ test // test is in the same line\n// test is in previous line", []token{ + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 24, Offset: 24}, + EndPos: ast.Position{Line: 1, Column: 24, Offset: 24}, + }, + }, + Source: "\n", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 18, Offset: 43}, + EndPos: ast.Position{Line: 2, Column: 18, Offset: 43}, + }, + }, + Source: " ", + }, { Token: Token{ Type: TokenIdentifier, @@ -888,9 +910,40 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 2, Column: 19, Offset: 44}, EndPos: ast.Position{Line: 2, Column: 22, Offset: 47}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in next line ")), + ast.NewComment(nil, []byte("/* test is here */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in the same line")), + }, + }, }, Source: "test", }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + EndPos: ast.Position{Line: 2, Column: 23, Offset: 48}, + }, + }, + Source: " ", + }, + { + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ContainsNewline: true}, + Range: ast.Range{ + StartPos: ast.Position{Line: 2, Column: 51, Offset: 76}, + EndPos: ast.Position{Line: 2, Column: 51, Offset: 76}, + }, + }, + Source: "\n", + }, { Token: Token{ Type: TokenEOF, @@ -898,6 +951,11 @@ func TestLexBasic(t *testing.T) { StartPos: ast.Position{Line: 3, Column: 27, Offset: 104}, EndPos: ast.Position{Line: 3, Column: 27, Offset: 104}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// test is in previous line")), + }, + }, }, }, }, @@ -2625,6 +2683,19 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/* test foo /* bar */ asd */ `, []token{ + { + Source: " ", + Token: Token{ + Type: TokenSpace, + SpaceOrError: Space{ + ContainsNewline: false, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 1, Column: 28, Offset: 28}, + EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, + }, + }, + }, { Token: Token{ Type: TokenEOF, @@ -2632,6 +2703,11 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, EndPos: ast.Position{Line: 1, Column: 30, Offset: 30}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/* test foo /* bar */ asd */")), + }, + }, }, }, }, @@ -2642,26 +2718,6 @@ func TestLexBlockComment(t *testing.T) { testLex(t, `/**/`, []token{ - { - Token: Token{ - Type: TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - }, - Source: "/*", - }, - { - Token: Token{ - Type: TokenBlockCommentEnd, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 2, Offset: 2}, - EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, - }, - }, - Source: "*/", - }, { Token: Token{ Type: TokenEOF, @@ -2669,6 +2725,11 @@ func TestLexBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 4, Offset: 4}, EndPos: ast.Position{Line: 1, Column: 4, Offset: 4}, }, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/**/")), + }, + }, }, }, }, @@ -3569,6 +3630,11 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// bar ")), + }, + }, }, Source: "foo", }, @@ -3624,6 +3690,11 @@ func TestLexLineComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 3, Offset: 3}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// bar ")), + }, + }, }, Source: "foo", }, diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 3e3e29fdd0..3525d2d4e2 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -360,7 +360,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { l.endOffset = beforeSlashOffset l.endOffset = starOffset } - return blockCommentState(nesting + 1) + return blockCommentState(l, nesting+1) } case '*': @@ -371,7 +371,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { l.endOffset = beforeStarOffset l.endOffset = slashOffset } - return blockCommentState(nesting - 1) + return blockCommentState(l, nesting-1) } return blockCommentState(l, nesting) diff --git a/parser/parser.go b/parser/parser.go index 168266bf87..b29fb1076f 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -508,11 +508,11 @@ func (p *parser) nonReservedIdentifier(errMsgContext string) (ast.Identifier, er } func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { - return ast.NewIdentifierWithComments( + return ast.NewIdentifier( p.memoryGauge, string(p.tokenSource(token)), token.StartPos, - token.Comments, + // TODO(preserve-comments): Handle comments for identifiers (attach them to parent structs?) ) } From 3707776ef5a17e37253c5172b72382c0f677ae14 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 10 Jun 2025 21:42:25 +0200 Subject: [PATCH 53/84] fix pooled lexer reset for comments & other smaller stuff --- ast/statement.go | 2 +- interpreter/memory_metering_test.go | 2 +- old_parser/expression_test.go | 1 - parser/lexer/lexer.go | 11 +---------- parser/statement_test.go | 11 ++++++----- 5 files changed, 9 insertions(+), 18 deletions(-) diff --git a/ast/statement.go b/ast/statement.go index fcbcbae582..37cb02ed02 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -214,7 +214,7 @@ type IfStatement struct { Else *Block StartPos Position `json:"-"` // Comments.Leading comments that appear before `if` keyword - Comments Comments + Comments Comments `json:"-"` } var _ Element = &IfStatement{} diff --git a/interpreter/memory_metering_test.go b/interpreter/memory_metering_test.go index cc3dfffdbd..21b3cda135 100644 --- a/interpreter/memory_metering_test.go +++ b/interpreter/memory_metering_test.go @@ -7755,7 +7755,7 @@ func TestInterpretTokenMetering(t *testing.T) { _, err := inter.Invoke("main") require.NoError(t, err) - assert.Equal(t, uint64(10), meter.getMemory(common.MemoryKindTypeToken)) + assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindTypeToken)) assert.Equal(t, uint64(6), meter.getMemory(common.MemoryKindSpaceToken)) assert.Equal(t, uint64(0), meter.getMemory(common.MemoryKindRawString)) }) diff --git a/old_parser/expression_test.go b/old_parser/expression_test.go index b3258f7b4f..556d6e5b79 100644 --- a/old_parser/expression_test.go +++ b/old_parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index d9c0a12a32..075a199914 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -149,6 +149,7 @@ func (l *lexer) clear() { l.tokenCount = 0 l.mode = lexerModeNormal l.openBrackets = 0 + l.currentComments = nil } func (l *lexer) Reclaim() { @@ -482,16 +483,6 @@ func IsIdentifierRune(r rune) bool { r == '_' } -func IsValidIdentifier(s string) bool { - for _, r := range s { - if !IsIdentifierRune(r) { - return false - } - } - - return true -} - func (l *lexer) scanLineComment() { // lookahead is already lexed. // parse more, if any diff --git a/parser/statement_test.go b/parser/statement_test.go index cf2c8bf99c..a18ed44d6f 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -1051,11 +1051,12 @@ for /* after for */ x /* before in */ in /* after in */ y { } Identifier: ast.Identifier{ Identifier: "x", Pos: ast.Position{Line: 3, Column: 20, Offset: 35}, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("/* before in */")), - }, - }, + // TODO(preserve-comments): Handle identifier comments + //Comments: ast.Comments{ + // Trailing: []*ast.Comment{ + // ast.NewComment(nil, []byte("/* before in */")), + // }, + //}, }, Value: &ast.IdentifierExpression{ Identifier: ast.Identifier{ From ef73932a1bfe350927d62aec0ddc37cfc6b8b47e Mon Sep 17 00:00:00 2001 From: Bart Date: Wed, 16 Jul 2025 21:04:22 +0200 Subject: [PATCH 54/84] fix leading inline comments for parameters --- parser/function.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parser/function.go b/parser/function.go index f335c3d7a0..381cf684a1 100644 --- a/parser/function.go +++ b/parser/function.go @@ -52,7 +52,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL p.next() expectParameter := true - + var commaToken lexer.Token var atEnd bool progress := p.newProgress() @@ -71,6 +71,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL if err != nil { return nil, err } + parameter.Comments.Leading = append(parameter.Comments.Leading, commaToken.Comments.Trailing...) parameters = append(parameters, parameter) expectParameter = false @@ -83,6 +84,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL ) } // Skip the comma + commaToken = p.current p.next() expectParameter = true From 60eddf3c41c604674c3c14eb8590f99481830777 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 17 Jul 2025 07:01:23 +0800 Subject: [PATCH 55/84] fix access token comments for variable declaration --- ast/variable_declaration.go | 4 ++++ parser/declaration.go | 15 +++++++++------ parser/statement.go | 2 +- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/ast/variable_declaration.go b/ast/variable_declaration.go index 0b6d13ebd6..20e5db4cbd 100644 --- a/ast/variable_declaration.go +++ b/ast/variable_declaration.go @@ -218,6 +218,10 @@ func (d *VariableDeclaration) Doc() prettier.Doc { var doc prettier.Concat + if !d.Comments.IsEmpty() { + doc = append(doc, d.Comments.LeadingDoc()) + } + if d.Access != AccessNotSpecified { doc = append( doc, diff --git a/parser/declaration.go b/parser/declaration.go index b358a667a9..8f168f4da1 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -109,7 +109,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if purity != ast.FunctionPurityUnspecified { return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") } - return parseVariableDeclaration(p, access, accessPos) + return parseVariableDeclaration(p, access, accessPos, startComments) case KeywordFun: return parseFunctionDeclaration( @@ -522,6 +522,7 @@ func parseVariableDeclaration( p *parser, access ast.Access, accessPos *ast.Position, + startComments []*ast.Comment, ) (*ast.VariableDeclaration, error) { startToken := p.current @@ -581,6 +582,12 @@ func parseVariableDeclaration( } } + var leadingComments []*ast.Comment + leadingComments = append(leadingComments, startComments...) + leadingComments = append(leadingComments, startToken.Comments.PackToList()...) + leadingComments = append(leadingComments, identifierToken.PackToList()...) + leadingComments = append(leadingComments, transferToken.PackToList()...) + variableDeclaration := ast.NewVariableDeclaration( p.memoryGauge, access, @@ -593,10 +600,7 @@ func parseVariableDeclaration( secondTransfer, secondValue, ast.Comments{ - Leading: append( - append(startToken.PackToList(), identifierToken.PackToList()...), - transferToken.PackToList()..., - ), + Leading: leadingComments, }, ) @@ -1178,7 +1182,6 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy skipNewlines: true, }) - switch p.current.Type { case endTokenType, lexer.TokenEOF: diff --git a/parser/statement.go b/parser/statement.go index 2b7290131a..17f1d85cb7 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -345,7 +345,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil) + parseVariableDeclaration(p, ast.AccessNotSpecified, nil, nil) if err != nil { return nil, err } From 50ba77017dcd9b91d5999280a3744668150765f9 Mon Sep 17 00:00:00 2001 From: Bart Date: Thu, 17 Jul 2025 13:32:38 +0800 Subject: [PATCH 56/84] attach comments to identifier expressions --- ast/attachment_test.go | 6 ++++++ ast/expression.go | 3 +++ parser/expression.go | 1 + parser/type.go | 1 + sema/before_extractor.go | 2 +- 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/ast/attachment_test.go b/ast/attachment_test.go index 7413504078..f03de36c7c 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -210,6 +210,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "foo", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), Attachment: NewInvocationExpression( nil, @@ -220,6 +221,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "bar", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), []*TypeAnnotation{}, Arguments{}, @@ -285,6 +287,7 @@ func TestAttachExpression_Doc(t *testing.T) { "foo", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), Attachment: NewInvocationExpression( nil, @@ -295,6 +298,7 @@ func TestAttachExpression_Doc(t *testing.T) { "bar", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), []*TypeAnnotation{}, Arguments{}, @@ -345,6 +349,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { "baz", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), StartPos: Position{Offset: 1, Line: 2, Column: 3}, } @@ -406,6 +411,7 @@ func TestRemoveStatement_Doc(t *testing.T) { "baz", Position{Offset: 1, Line: 2, Column: 3}, ), + Comments{}, ), StartPos: Position{Offset: 1, Line: 2, Column: 3}, } diff --git a/ast/expression.go b/ast/expression.go index 313608020e..fb3b36590f 100644 --- a/ast/expression.go +++ b/ast/expression.go @@ -671,6 +671,7 @@ func (e DictionaryEntry) Doc() prettier.Doc { type IdentifierExpression struct { Identifier Identifier + Comments Comments } var _ Element = &IdentifierExpression{} @@ -679,11 +680,13 @@ var _ Expression = &IdentifierExpression{} func NewIdentifierExpression( gauge common.MemoryGauge, identifier Identifier, + comments Comments, ) *IdentifierExpression { common.UseMemory(gauge, common.IdentifierExpressionMemoryUsage) return &IdentifierExpression{ Identifier: identifier, + Comments: comments, } } diff --git a/parser/expression.go b/parser/expression.go index de6ceff870..504606d55a 100644 --- a/parser/expression.go +++ b/parser/expression.go @@ -840,6 +840,7 @@ func defineIdentifierExpression() { return ast.NewIdentifierExpression( p.memoryGauge, p.tokenToIdentifier(token), + token.Comments, ), nil }, }) diff --git a/parser/type.go b/parser/type.go index d5e89f94f5..401ec5d531 100644 --- a/parser/type.go +++ b/parser/type.go @@ -834,6 +834,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, var invokedExpression ast.Expression = ast.NewIdentifierExpression( p.memoryGauge, ty.Identifier, + ty.Comments, ) for _, nestedIdentifier := range ty.NestedIdentifiers { diff --git a/sema/before_extractor.go b/sema/before_extractor.go index b99842f5fc..034aa56eb0 100644 --- a/sema/before_extractor.go +++ b/sema/before_extractor.go @@ -76,7 +76,7 @@ func (e *BeforeExtractor) ExtractInvocation( ast.EmptyPosition, ) - newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier) + newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier, ast.Comments{}) extractedExpressions = append(extractedExpressions, ast.ExtractedExpression{ From 522daba7f0947fffc88c567bc24128ad6033052c Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 5 Aug 2025 17:49:00 +0700 Subject: [PATCH 57/84] remove legacy DocString fields --- ast/composite.go | 14 +- ast/composite_test.go | 6 +- ast/position.go | 1 - ast/variable_declaration.go | 4 - old_parser/declaration.go | 2 +- old_parser/expression.go | 1 + old_parser/type.go | 1 + parser/declaration.go | 430 +++++++++++++--------------- parser/declaration_test.go | 5 +- parser/function.go | 24 +- parser/lexer/token.go | 10 + parser/statement.go | 5 +- parser/transaction.go | 12 +- parser/type.go | 3 +- sema/check_composite_declaration.go | 8 +- 15 files changed, 245 insertions(+), 281 deletions(-) diff --git a/ast/composite.go b/ast/composite.go index a88ac44fa0..a2b71e4a5a 100644 --- a/ast/composite.go +++ b/ast/composite.go @@ -306,9 +306,7 @@ const ( type FieldDeclaration struct { TypeAnnotation *TypeAnnotation - // TODO(preserve-comments): Remove - DocString string - Identifier Identifier + Identifier Identifier Range Access Access VariableKind VariableKind @@ -379,7 +377,7 @@ func (d *FieldDeclaration) DeclarationMembers() *Members { } func (d *FieldDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *FieldDeclaration) MarshalJSON() ([]byte, error) { @@ -498,10 +496,10 @@ func (d *FieldDeclaration) IsNative() bool { // EnumCaseDeclaration type EnumCaseDeclaration struct { - DocString string Identifier Identifier StartPos Position `json:"-"` Access Access + Comments Comments } var _ Element = &EnumCaseDeclaration{} @@ -511,7 +509,7 @@ func NewEnumCaseDeclaration( memoryGauge common.MemoryGauge, access Access, identifier Identifier, - docString string, + comments Comments, startPos Position, ) *EnumCaseDeclaration { common.UseMemory(memoryGauge, common.EnumCaseDeclarationMemoryUsage) @@ -519,7 +517,7 @@ func NewEnumCaseDeclaration( return &EnumCaseDeclaration{ Access: access, Identifier: identifier, - DocString: docString, + Comments: comments, StartPos: startPos, } } @@ -559,7 +557,7 @@ func (d *EnumCaseDeclaration) DeclarationMembers() *Members { } func (d *EnumCaseDeclaration) DeclarationDocString() string { - return d.DocString + return d.Comments.LeadingDocString() } func (d *EnumCaseDeclaration) MarshalJSON() ([]byte, error) { diff --git a/ast/composite_test.go b/ast/composite_test.go index 02fde34874..ea2c670f44 100644 --- a/ast/composite_test.go +++ b/ast/composite_test.go @@ -51,7 +51,11 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { }, StartPos: Position{Offset: 7, Line: 8, Column: 9}, }, - DocString: "test", + Comments: Comments{ + Leading: []*Comment{ + NewComment(nil, []byte("///test")), + }, + }, Range: Range{ StartPos: Position{Offset: 10, Line: 11, Column: 12}, EndPos: Position{Offset: 13, Line: 14, Column: 15}, diff --git a/ast/position.go b/ast/position.go index 01e80967c2..73fa99191f 100644 --- a/ast/position.go +++ b/ast/position.go @@ -20,7 +20,6 @@ package ast import ( "fmt" - "github.com/onflow/cadence/common" ) diff --git a/ast/variable_declaration.go b/ast/variable_declaration.go index 20e5db4cbd..0b6d13ebd6 100644 --- a/ast/variable_declaration.go +++ b/ast/variable_declaration.go @@ -218,10 +218,6 @@ func (d *VariableDeclaration) Doc() prettier.Doc { var doc prettier.Concat - if !d.Comments.IsEmpty() { - doc = append(doc, d.Comments.LeadingDoc()) - } - if d.Access != AccessNotSpecified { doc = append( doc, diff --git a/old_parser/declaration.go b/old_parser/declaration.go index 95e59ee106..1015c6ccf5 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -1577,7 +1577,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - docString, + ast.Comments{}, startPos, ), nil } diff --git a/old_parser/expression.go b/old_parser/expression.go index 7f0cd69ea4..75491287ad 100644 --- a/old_parser/expression.go +++ b/old_parser/expression.go @@ -821,6 +821,7 @@ func defineIdentifierExpression() { return ast.NewIdentifierExpression( p.memoryGauge, p.tokenToIdentifier(token), + ast.Comments{}, ), nil } }, diff --git a/old_parser/type.go b/old_parser/type.go index 359af4787d..9a58b19f35 100644 --- a/old_parser/type.go +++ b/old_parser/type.go @@ -883,6 +883,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, var invokedExpression ast.Expression = ast.NewIdentifierExpression( p.memoryGauge, ty.Identifier, + ast.Comments{}, ) for _, nestedIdentifier := range ty.NestedIdentifiers { diff --git a/parser/declaration.go b/parser/declaration.go index 8f168f4da1..adcb46e7f7 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -67,16 +67,14 @@ func parseDeclarations(p *parser, endTokenType lexer.TokenType) (declarations [] } func parseDeclaration(p *parser) (ast.Declaration, error) { - var startComments []*ast.Comment - var access ast.Access = ast.AccessNotSpecified - var accessPos *ast.Position + var accessToken *lexer.Token purity := ast.FunctionPurityUnspecified - var purityPos *ast.Position + var purityToken *lexer.Token - var staticPos *ast.Position - var nativePos *ast.Position + var staticToken *lexer.Token + var nativeToken *lexer.Token staticModifierEnabled := p.config.StaticModifierEnabled nativeModifierEnabled := p.config.NativeModifierEnabled @@ -90,9 +88,9 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { switch p.current.Type { case lexer.TokenPragma: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for pragma") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for pragma") } - err := rejectAllModifiers(p, access, accessPos, staticPos, nativePos, common.DeclarationKindPragma) + err := rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, common.DeclarationKindPragma) if err != nil { return nil, err } @@ -102,123 +100,122 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindVariable) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindVariable) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for variable") } - return parseVariableDeclaration(p, access, accessPos, startComments) + return parseVariableDeclaration(p, access, accessToken) case KeywordFun: return parseFunctionDeclaration( p, false, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, - startComments, + purityToken, + staticToken, + nativeToken, ) case KeywordImport: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindImport) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindImport) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for import") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for import") } return parseImportDeclaration(p) case KeywordEvent: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEvent) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEvent) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for event") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for event") } - return parseEventDeclaration(p, access, accessPos, startComments) + return parseEventDeclaration(p, access, accessToken) case KeywordStruct: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindStructure) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindStructure) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for struct") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for struct") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordResource: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindResource) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindResource) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for resource") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for resource") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEntitlement: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEntitlement) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) + return parseEntitlementOrMappingDeclaration(p, access, accessToken) case KeywordAttachment: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindAttachment) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindAttachment) if err != nil { return nil, err } - return parseAttachmentDeclaration(p, access, accessPos, startComments) + return parseAttachmentDeclaration(p, access, accessToken) case KeywordContract: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindContract) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindContract) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for contract") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for contract") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEnum: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEnum) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEnum) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for enum") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for enum") } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordTransaction: - err := rejectAllModifiers(p, access, accessPos, staticPos, nativePos, common.DeclarationKindTransaction) + err := rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, common.DeclarationKindTransaction) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for transaction") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for transaction") } - return parseTransactionDeclaration(p, startComments) + return parseTransactionDeclaration(p) case KeywordView: if purity != ast.FunctionPurityUnspecified { p.report(p.syntaxError("invalid second view modifier")) } - pos := p.current.StartPos - purityPos = &pos + tok := p.current + purityToken = &tok purity = parsePurityAnnotation(p) continue @@ -237,16 +234,16 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if access != ast.AccessNotSpecified { return nil, p.syntaxError("invalid second access modifier") } - if staticModifierEnabled && staticPos != nil { + if staticModifierEnabled && staticToken != nil { return nil, p.syntaxError("invalid access modifier after static modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { return nil, p.syntaxError("invalid access modifier after native modifier") } - pos := p.current.StartPos - accessPos = &pos + tok := p.current + accessToken = &tok var err error - access, startComments, err = parseAccess(p) + access, err = parseAccess(p) if err != nil { return nil, err } @@ -258,14 +255,14 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { break } - if staticPos != nil { + if staticToken != nil { return nil, p.syntaxError("invalid second static modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { return nil, p.syntaxError("invalid static modifier after native modifier") } - pos := p.current.StartPos - staticPos = &pos + tok := p.current + staticToken = &tok p.next() continue @@ -274,11 +271,11 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { break } - if nativePos != nil { + if nativeToken != nil { return nil, p.syntaxError("invalid second native modifier") } - pos := p.current.StartPos - nativePos = &pos + tok := p.current + nativeToken = &tok p.next() continue } @@ -430,22 +427,21 @@ func parseEntitlementList(p *parser) (ast.EntitlementSet, error) { // : 'access(self)' // | 'access(all)' ( '(' 'set' ')' )? // | 'access' '(' ( 'self' | 'contract' | 'account' | 'all' | entitlementList ) ')' -func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { +func parseAccess(p *parser) (ast.Access, error) { switch string(p.currentTokenSource()) { case KeywordAccess: - accessTokenComments := p.current.Comments.PackToList() // Skip the `access` keyword p.nextSemanticToken() _, err := p.mustOne(lexer.TokenParenOpen) if err != nil { - return ast.AccessNotSpecified, accessTokenComments, err + return ast.AccessNotSpecified, err } p.skipSpace() if !p.current.Is(lexer.TokenIdentifier) { - return ast.AccessNotSpecified, accessTokenComments, p.syntaxError( + return ast.AccessNotSpecified, p.syntaxError( "expected keyword %s, got %s", enumeratedAccessModifierKeywords, p.current.Type, @@ -484,7 +480,7 @@ func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { entitlementMapName, err := parseNominalType(p, lowestBindingPower) if err != nil { - return ast.AccessNotSpecified, accessTokenComments, err + return ast.AccessNotSpecified, err } access = ast.NewMappedAccess(entitlementMapName, keywordPos) @@ -493,20 +489,20 @@ func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { default: entitlements, err := parseEntitlementList(p) if err != nil { - return ast.AccessNotSpecified, accessTokenComments, err + return ast.AccessNotSpecified, err } access = ast.NewEntitlementAccess(entitlements) } _, err = p.mustOne(lexer.TokenParenClose) if err != nil { - return ast.AccessNotSpecified, accessTokenComments, err + return ast.AccessNotSpecified, err } - return access, accessTokenComments, nil + return access, nil default: - return ast.AccessNotSpecified, nil, errors.NewUnreachableError() + return ast.AccessNotSpecified, errors.NewUnreachableError() } } @@ -521,14 +517,12 @@ func parseAccess(p *parser) (ast.Access, []*ast.Comment, error) { func parseVariableDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, ) (*ast.VariableDeclaration, error) { startToken := p.current - startPos := startToken.StartPos - if accessPos != nil { - startPos = *accessPos + if accessToken != nil { + startToken = *accessToken } isLet := string(p.currentTokenSource()) == KeywordLet @@ -583,7 +577,6 @@ func parseVariableDeclaration( } var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) leadingComments = append(leadingComments, startToken.Comments.PackToList()...) leadingComments = append(leadingComments, identifierToken.PackToList()...) leadingComments = append(leadingComments, transferToken.PackToList()...) @@ -596,7 +589,7 @@ func parseVariableDeclaration( typeAnnotation, value, transfer, - startPos, + startToken.StartPos, secondTransfer, secondValue, ast.Comments{ @@ -959,13 +952,11 @@ func parseHexadecimalLocation(p *parser) common.AddressLocation { func parseEventDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, ) (*ast.CompositeDeclaration, error) { startToken := p.current - startPos := startToken.StartPos - if accessPos != nil { - startPos = *accessPos + if accessToken != nil { + startToken = *accessToken } // Skip the `event` keyword @@ -1011,10 +1002,6 @@ func parseEventDeclaration( }, ) - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) - return ast.NewCompositeDeclaration( p.memoryGauge, access, @@ -1024,11 +1011,11 @@ func parseEventDeclaration( members, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, parameterList.EndPos, ), ast.Comments{ - Leading: leadingComments, + Leading: startToken.Comments.Leading, }, ), nil } @@ -1065,12 +1052,11 @@ func parseCompositeKind(p *parser) common.CompositeKind { func parseFieldWithVariableKind( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, ) (*ast.FieldDeclaration, error) { - startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) + startToken := lexer.EarliestToken(p.current, accessToken, staticToken, nativeToken) var variableKind ast.VariableKind switch string(p.currentTokenSource()) { @@ -1109,18 +1095,18 @@ func parseFieldWithVariableKind( return ast.NewFieldDeclaration( p.memoryGauge, access, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, variableKind, identifier, typeAnnotation, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, typeAnnotation.EndPosition(p.memoryGauge), ), ast.Comments{ - Leading: startComments, + Leading: startToken.Comments.Leading, }, ), nil } @@ -1229,13 +1215,11 @@ func parseEntitlementMappingsAndInclusions(p *parser, endTokenType lexer.TokenTy func parseEntitlementOrMappingDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, ) (ast.Declaration, error) { startToken := p.current - startPos := startToken.StartPos - if accessPos != nil { - startPos = *accessPos + if accessToken != nil { + startToken = *accessToken } // Skip the `entitlement` keyword @@ -1268,9 +1252,9 @@ func parseEntitlementOrMappingDeclaration( } p.nextSemanticToken() - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) + comments := ast.Comments{ + Leading: startToken.Comments.Leading, + } if isMapping { _, err = p.mustOne(lexer.TokenBraceOpen) @@ -1290,7 +1274,7 @@ func parseEntitlementOrMappingDeclaration( } declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) @@ -1300,14 +1284,12 @@ func parseEntitlementOrMappingDeclaration( identifier, elements, declarationRange, - ast.Comments{ - Leading: leadingComments, - }, + comments, ), nil } else { declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, identifier.EndPosition(p.memoryGauge), ) @@ -1316,9 +1298,7 @@ func parseEntitlementOrMappingDeclaration( access, identifier, declarationRange, - ast.Comments{ - Leading: leadingComments, - }, + comments, ), nil } } @@ -1360,14 +1340,12 @@ func parseConformances(p *parser) ([]*ast.NominalType, error) { func parseCompositeOrInterfaceDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, ) (ast.Declaration, error) { startToken := p.current - startPos := startToken.StartPos - if accessPos != nil { - startPos = *accessPos + if accessToken != nil { + startToken = *accessToken } compositeKind := parseCompositeKind(p) @@ -1445,13 +1423,13 @@ func parseCompositeOrInterfaceDeclaration( declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) + comments := ast.Comments{ + Leading: startToken.Comments.Leading, + } if isInterface { return ast.NewInterfaceDeclaration( @@ -1462,9 +1440,7 @@ func parseCompositeOrInterfaceDeclaration( conformances, members, declarationRange, - ast.Comments{ - Leading: leadingComments, - }, + comments, ), nil } else { return ast.NewCompositeDeclaration( @@ -1475,9 +1451,7 @@ func parseCompositeOrInterfaceDeclaration( conformances, members, declarationRange, - ast.Comments{ - Leading: leadingComments, - }, + comments, ), nil } } @@ -1485,14 +1459,12 @@ func parseCompositeOrInterfaceDeclaration( func parseAttachmentDeclaration( p *parser, access ast.Access, - accessPos *ast.Position, - startComments []*ast.Comment, + accessToken *lexer.Token, ) (ast.Declaration, error) { startToken := p.current - startPos := startToken.StartPos - if accessPos != nil { - startPos = *accessPos + if accessToken != nil { + startToken = *accessToken } // Skip the `attachment` keyword @@ -1563,14 +1535,10 @@ func parseAttachmentDeclaration( declarationRange := ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endToken.EndPos, ) - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) - return ast.NewAttachmentDeclaration( p.memoryGauge, access, @@ -1580,7 +1548,7 @@ func parseAttachmentDeclaration( members, declarationRange, ast.Comments{ - Leading: leadingComments, + Leading: startToken.Comments.Leading, }, ), nil } @@ -1639,18 +1607,16 @@ func parseMembersAndNestedDeclarations(p *parser, endTokenType lexer.TokenType) // | enumCase // | pragmaDeclaration func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { - var startComments []*ast.Comment - const functionBlockIsOptional = true var access ast.Access = ast.AccessNotSpecified - var accessPos *ast.Position + var accessToken *lexer.Token purity := ast.FunctionPurityUnspecified - var purityPos *ast.Position + var purityToken *lexer.Token - var staticPos *ast.Position - var nativePos *ast.Position + var staticToken *lexer.Token + var nativeToken *lexer.Token var previousIdentifierToken *lexer.Token @@ -1679,109 +1645,107 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for variable") } return parseFieldWithVariableKind( p, access, - accessPos, - staticPos, - nativePos, - startComments, + accessToken, + staticToken, + nativeToken, ) case KeywordCase: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for enum case") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for enum case") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEnumCase) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEnumCase) if err != nil { return nil, err } - return parseEnumCase(p, access, accessPos) + return parseEnumCase(p, access, accessToken) case KeywordFun: return parseFunctionDeclaration( p, functionBlockIsOptional, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, - startComments, + purityToken, + staticToken, + nativeToken, ) case KeywordEvent: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for event") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for event") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEvent) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEvent) if err != nil { return nil, err } - return parseEventDeclaration(p, access, accessPos, startComments) + return parseEventDeclaration(p, access, accessToken) case KeywordStruct: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for struct") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for struct") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindStructure) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindStructure) if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordResource: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for resource") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for resource") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindResource) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindResource) if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordContract: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for contract") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for contract") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindContract) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindContract) if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEntitlement: - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEntitlement) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEntitlement) if err != nil { return nil, err } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for entitlement") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for entitlement") } - return parseEntitlementOrMappingDeclaration(p, access, accessPos, startComments) + return parseEntitlementOrMappingDeclaration(p, access, accessToken) case KeywordEnum: if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for enum") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for enum") } - err := rejectStaticAndNativeModifiers(p, staticPos, nativePos, common.DeclarationKindEnum) + err := rejectStaticAndNativeModifiers(p, staticToken, nativeToken, common.DeclarationKindEnum) if err != nil { return nil, err } - return parseCompositeOrInterfaceDeclaration(p, access, accessPos, startComments) + return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordAttachment: - return parseAttachmentDeclaration(p, access, accessPos, startComments) + return parseAttachmentDeclaration(p, access, accessToken) case KeywordView: if purity != ast.FunctionPurityUnspecified { return nil, p.syntaxError("invalid second view modifier") } - pos := p.current.StartPos - purityPos = &pos + tok := p.current + purityToken = &tok purity = parsePurityAnnotation(p) continue @@ -1800,16 +1764,16 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { if access != ast.AccessNotSpecified { return nil, p.syntaxError("invalid second access modifier") } - if staticModifierEnabled && staticPos != nil { + if staticModifierEnabled && staticToken != nil { return nil, p.syntaxError("invalid access modifier after static modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { return nil, p.syntaxError("invalid access modifier after native modifier") } - pos := p.current.StartPos - accessPos = &pos + tok := p.current + accessToken = &tok var err error - access, startComments, err = parseAccess(p) + access, err = parseAccess(p) if err != nil { return nil, err } @@ -1820,14 +1784,14 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { break } - if staticPos != nil { + if staticToken != nil { return nil, p.syntaxError("invalid second static modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { return nil, p.syntaxError("invalid static modifier after native modifier") } - pos := p.current.StartPos - staticPos = &pos + tok := p.current + staticToken = &tok p.next() continue @@ -1836,11 +1800,11 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { break } - if nativePos != nil { + if nativeToken != nil { return nil, p.syntaxError("invalid second native modifier") } - pos := p.current.StartPos - nativePos = &pos + tok := p.current + nativeToken = &tok p.next() continue } @@ -1865,7 +1829,7 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { previousIdentifierToken.Type, ) } - err := rejectAllModifiers(p, access, accessPos, staticPos, nativePos, common.DeclarationKindPragma) + err := rejectAllModifiers(p, access, accessToken, staticToken, nativeToken, common.DeclarationKindPragma) if err != nil { return nil, err } @@ -1876,15 +1840,17 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { return nil, p.syntaxError("unexpected %s", p.current.Type) } if purity != ast.FunctionPurityUnspecified { - return nil, NewSyntaxError(*purityPos, "invalid view modifier for variable") + return nil, NewSyntaxError(purityToken.StartPos, "invalid view modifier for variable") } - identifier := p.tokenToIdentifier(*previousIdentifierToken) + identifierToken := *previousIdentifierToken + identifier := p.tokenToIdentifier(identifierToken) return parseFieldDeclarationWithoutVariableKind( p, access, - accessPos, - staticPos, - nativePos, + accessToken, + staticToken, + nativeToken, + identifierToken, identifier, ) @@ -1897,11 +1863,11 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { p, functionBlockIsOptional, access, - accessPos, + accessToken, purity, - purityPos, - staticPos, - nativePos, + purityToken, + staticToken, + nativeToken, *previousIdentifierToken, ) } @@ -1915,28 +1881,28 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { func rejectAllModifiers( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, kind common.DeclarationKind, ) error { if access != ast.AccessNotSpecified { - return NewSyntaxError(*accessPos, "invalid access modifier for %s", kind.Name()) + return NewSyntaxError(accessToken.StartPos, "invalid access modifier for %s", kind.Name()) } - return rejectStaticAndNativeModifiers(p, staticPos, nativePos, kind) + return rejectStaticAndNativeModifiers(p, staticToken, nativeToken, kind) } func rejectStaticAndNativeModifiers( p *parser, - staticPos *ast.Position, - nativePos *ast.Position, + staticToken *lexer.Token, + nativeToken *lexer.Token, kind common.DeclarationKind, ) error { - if p.config.StaticModifierEnabled && staticPos != nil { - return NewSyntaxError(*staticPos, "invalid static modifier for %s", kind.Name()) + if p.config.StaticModifierEnabled && staticToken != nil { + return NewSyntaxError(staticToken.StartPos, "invalid static modifier for %s", kind.Name()) } - if p.config.NativeModifierEnabled && nativePos != nil { - return NewSyntaxError(*nativePos, "invalid native modifier for %s", kind.Name()) + if p.config.NativeModifierEnabled && nativeToken != nil { + return NewSyntaxError(nativeToken.StartPos, "invalid native modifier for %s", kind.Name()) } return nil } @@ -1944,12 +1910,13 @@ func rejectStaticAndNativeModifiers( func parseFieldDeclarationWithoutVariableKind( p *parser, access ast.Access, - accessPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, + accessToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, + identifierToken lexer.Token, identifier ast.Identifier, ) (*ast.FieldDeclaration, error) { - startPos := ast.EarliestPosition(identifier.Pos, accessPos, staticPos, nativePos) + startToken := lexer.EarliestToken(identifierToken, accessToken, staticToken, nativeToken) _, err := p.mustOne(lexer.TokenColon) if err != nil { @@ -1966,17 +1933,19 @@ func parseFieldDeclarationWithoutVariableKind( return ast.NewFieldDeclaration( p.memoryGauge, access, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, ast.VariableKindNotSpecified, identifier, typeAnnotation, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, typeAnnotation.EndPosition(p.memoryGauge), ), - ast.Comments{}, + ast.Comments{ + Leading: startToken.Comments.Leading, + }, ), nil } @@ -1984,16 +1953,16 @@ func parseSpecialFunctionDeclaration( p *parser, functionBlockIsOptional bool, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, purity ast.FunctionPurity, - purityPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, + purityToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, identifierToken lexer.Token, ) (*ast.SpecialFunctionDeclaration, error) { identifier := p.tokenToIdentifier(identifierToken) - startPos := ast.EarliestPosition(identifier.Pos, accessPos, purityPos, staticPos, nativePos) + startToken := lexer.EarliestToken(identifierToken, accessToken, purityToken, staticToken, nativeToken) parameterList, returnTypeAnnotation, functionBlock, err := parseFunctionParameterListAndRest(p, functionBlockIsOptional) @@ -2035,16 +2004,16 @@ func parseSpecialFunctionDeclaration( p.memoryGauge, access, purity, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, identifier, nil, parameterList, nil, functionBlock, - startPos, + startToken.StartPos, ast.Comments{ - Leading: identifierToken.Leading, + Leading: startToken.Comments.Leading, }, ), ), nil @@ -2056,14 +2025,11 @@ func parseSpecialFunctionDeclaration( func parseEnumCase( p *parser, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, ) (*ast.EnumCaseDeclaration, error) { - // TODO(preserve-comments): Implement - var docString string - - startPos := p.current.StartPos - if accessPos != nil { - startPos = *accessPos + startToken := p.current + if accessToken != nil { + startToken = *accessToken } // Skip the `enum` keyword @@ -2083,7 +2049,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - docString, - startPos, + ast.Comments{Leading: startToken.Comments.PackToList()}, + startToken.StartPos, ), nil } diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 73c32a1027..44c9168501 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -340,6 +340,7 @@ var x = /* Before 1 */ 1 // After 1 ) }) + // TODO: Fix these cases t.Run("with purity", func(t *testing.T) { t.Parallel() @@ -1874,7 +1875,7 @@ func TestParseAccess(t *testing.T) { nil, []byte(input), func(p *parser) (ast.Access, error) { - access, _, err := parseAccess(p) + access, err := parseAccess(p) return access, err }, Config{}, @@ -3201,7 +3202,6 @@ func TestParseFieldWithVariableKind(t *testing.T) { nil, nil, nil, - nil, ) }, Config{}, @@ -6723,7 +6723,6 @@ func TestParseStructure(t *testing.T) { }, IsResource: false, }, - DocString: "", Identifier: ast.Identifier{ Identifier: "foo", Pos: ast.Position{ diff --git a/parser/function.go b/parser/function.go index 381cf684a1..0e352c64f5 100644 --- a/parser/function.go +++ b/parser/function.go @@ -360,15 +360,13 @@ func parseFunctionDeclaration( p *parser, functionBlockIsOptional bool, access ast.Access, - accessPos *ast.Position, + accessToken *lexer.Token, purity ast.FunctionPurity, - purityPos *ast.Position, - staticPos *ast.Position, - nativePos *ast.Position, - startComments []*ast.Comment, + purityToken *lexer.Token, + staticToken *lexer.Token, + nativeToken *lexer.Token, ) (*ast.FunctionDeclaration, error) { - startToken := p.current - startPos := ast.EarliestPosition(startToken.StartPos, accessPos, purityPos, staticPos, nativePos) + startToken := lexer.EarliestToken(p.current, accessToken, purityToken, staticToken, nativeToken) // Skip the `fun` keyword p.nextSemanticToken() @@ -399,24 +397,20 @@ func parseFunctionDeclaration( return nil, err } - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) - return ast.NewFunctionDeclaration( p.memoryGauge, access, purity, - staticPos != nil, - nativePos != nil, + staticToken != nil, + nativeToken != nil, identifier, typeParameterList, parameterList, returnTypeAnnotation, functionBlock, - startPos, + startToken.StartPos, ast.Comments{ - Leading: leadingComments, + Leading: startToken.Comments.Leading, }, ), nil } diff --git a/parser/lexer/token.go b/parser/lexer/token.go index 2d0e5dcf84..8bf7b47289 100644 --- a/parser/lexer/token.go +++ b/parser/lexer/token.go @@ -46,3 +46,13 @@ func (t Token) Equal(other Token) bool { // ignore comments, since they should not be treated as source code return t.Type == other.Type && t.Range == other.Range && t.SpaceOrError == other.SpaceOrError } + +func EarliestToken(t Token, ts ...*Token) (earliest Token) { + earliest = t + for _, tok := range ts { + if tok != nil && tok.StartPos.Compare(earliest.StartPos) < 0 { + earliest = *tok + } + } + return +} diff --git a/parser/statement.go b/parser/statement.go index 17f1d85cb7..d135247194 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -279,7 +279,8 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { var err error switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: - endToken = &p.current + tok := p.current + endToken = &tok break default: if !sawNewLine { @@ -345,7 +346,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { switch string(p.currentTokenSource()) { case KeywordLet, KeywordVar: variableDeclaration, err = - parseVariableDeclaration(p, ast.AccessNotSpecified, nil, nil) + parseVariableDeclaration(p, ast.AccessNotSpecified, nil) if err != nil { return nil, err } diff --git a/parser/transaction.go b/parser/transaction.go index a8b3390231..5992858dc9 100644 --- a/parser/transaction.go +++ b/parser/transaction.go @@ -40,10 +40,9 @@ import ( // | /* no execute or postConditions */ // ) // '}' -func parseTransactionDeclaration(p *parser, startComments []*ast.Comment) (*ast.TransactionDeclaration, error) { +func parseTransactionDeclaration(p *parser) (*ast.TransactionDeclaration, error) { startToken := p.current - startPos := startToken.StartPos // Skip the `transaction` keyword p.nextSemanticToken() @@ -199,10 +198,6 @@ func parseTransactionDeclaration(p *parser, startComments []*ast.Comment) (*ast. } } - var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startComments...) - leadingComments = append(leadingComments, startToken.Comments.Leading...) - return ast.NewTransactionDeclaration( p.memoryGauge, parameterList, @@ -213,11 +208,11 @@ func parseTransactionDeclaration(p *parser, startComments []*ast.Comment) (*ast. execute, ast.NewRange( p.memoryGauge, - startPos, + startToken.StartPos, endPos, ), ast.Comments{ - Leading: leadingComments, + Leading: startToken.Comments.Leading, }, ), nil } @@ -249,7 +244,6 @@ func parseTransactionFields(p *parser) (fields []*ast.FieldDeclaration, err erro nil, nil, nil, - nil, ) if err != nil { return nil, err diff --git a/parser/type.go b/parser/type.go index 401ec5d531..1273cd32b1 100644 --- a/parser/type.go +++ b/parser/type.go @@ -179,7 +179,8 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, // Skip the dot p.next() - nestedToken = &p.current + tok := p.current + nestedToken = &tok if !nestedToken.Is(lexer.TokenIdentifier) { return nil, p.syntaxError( diff --git a/sema/check_composite_declaration.go b/sema/check_composite_declaration.go index 7463754f52..44f8ee65c1 100644 --- a/sema/check_composite_declaration.go +++ b/sema/check_composite_declaration.go @@ -1083,7 +1083,7 @@ func (checker *Checker) declareEnumConstructor( TypeAnnotation: memberCaseTypeAnnotation, DeclarationKind: common.DeclarationKindField, VariableKind: ast.VariableKindConstant, - DocString: enumCase.DocString, + DocString: enumCase.DeclarationDocString(), }, ) @@ -1092,7 +1092,7 @@ func (checker *Checker) declareEnumConstructor( checker.recordFieldDeclarationOrigin( enumCase.Identifier, compositeType, - enumCase.DocString, + enumCase.DeclarationDocString(), ) } } @@ -1827,7 +1827,7 @@ func (checker *Checker) defaultMembersAndOrigins( DeclarationKind: declarationKind, TypeAnnotation: fieldTypeAnnotation, VariableKind: field.VariableKind, - DocString: field.DocString, + DocString: field.DeclarationDocString(), }) if checker.PositionInfo != nil && origins != nil { @@ -1835,7 +1835,7 @@ func (checker *Checker) defaultMembersAndOrigins( checker.recordFieldDeclarationOrigin( field.Identifier, fieldTypeAnnotation.Type, - field.DocString, + field.DeclarationDocString(), ) } From 19ebdaba3ec21bb3904c5cc02b8ccde06aa86d92 Mon Sep 17 00:00:00 2001 From: Bart Date: Tue, 5 Aug 2025 23:23:15 +0700 Subject: [PATCH 58/84] fix tests --- parser/statement_test.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/parser/statement_test.go b/parser/statement_test.go index a18ed44d6f..b601969eb9 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -98,6 +98,8 @@ return // After return Comments: ast.Comments{ Leading: []*ast.Comment{ ast.NewComment(nil, []byte("// Before return")), + }, + Trailing: []*ast.Comment{ ast.NewComment(nil, []byte("// After return")), }, }, @@ -1538,16 +1540,16 @@ func TestParseRemoveAttachmentStatement(t *testing.T) { Attachment: &ast.NominalType{ Identifier: ast.Identifier{ Identifier: "A", - Pos: ast.Position{Line: 3, Column: 7, Offset: 25}, + Pos: ast.Position{Line: 1, Column: 7, Offset: 7}, }, }, Value: &ast.IdentifierExpression{ Identifier: ast.Identifier{ Identifier: "b", - Pos: ast.Position{Line: 5, Column: 5, Offset: 47}, + Pos: ast.Position{Line: 1, Column: 14, Offset: 14}, }, }, - StartPos: ast.Position{Line: 3, Column: 0, Offset: 18}, + StartPos: ast.Position{Line: 1, Column: 0, Offset: 0}, }, }, result, From 14834e4c604b3be9a6f07e25a11876ac9944163e Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 18 Nov 2025 20:59:28 +0100 Subject: [PATCH 59/84] fixed all lint issues --- ast/comments.go | 5 +- ast/entitlement_declaration_test.go | 2 - ast/function_declaration.go | 1 + ast/position.go | 7 +- ast/statement.go | 17 +-- ast/transaction_declaration_test.go | 2 +- bbq/compiler/desugar.go | 74 +++++++++---- old_parser/comment.go | 1 - old_parser/expression_test.go | 1 - old_parser/function.go | 4 +- old_parser/statement.go | 4 +- parser/comment.go | 1 - parser/comment_test.go | 166 +++++++++++++++++++++++++--- parser/declaration.go | 10 +- parser/declaration_test.go | 13 +-- parser/errors.go | 6 +- parser/expression_test.go | 141 +---------------------- parser/function.go | 5 +- parser/statement.go | 4 +- parser/statement_test.go | 142 ------------------------ 20 files changed, 234 insertions(+), 372 deletions(-) delete mode 100644 old_parser/comment.go delete mode 100644 parser/comment.go diff --git a/ast/comments.go b/ast/comments.go index 75822425e1..ddeae5f185 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -2,8 +2,9 @@ package ast import ( "bytes" - "github.com/onflow/cadence/common" "strings" + + "github.com/onflow/cadence/common" ) type Comments struct { @@ -11,6 +12,8 @@ type Comments struct { Trailing []*Comment `json:"-"` } +var EmptyComments = Comments{} + func (c Comments) PackToList() []*Comment { var comments []*Comment comments = append(comments, c.Leading...) diff --git a/ast/entitlement_declaration_test.go b/ast/entitlement_declaration_test.go index b3dbb16696..2108a380d6 100644 --- a/ast/entitlement_declaration_test.go +++ b/ast/entitlement_declaration_test.go @@ -289,7 +289,6 @@ func TestEntitlementMappingDeclaration_Doc(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, @@ -363,7 +362,6 @@ func TestEntitlementMappingDeclaration_String(t *testing.T) { Identifier: "AB", Pos: Position{Offset: 1, Line: 2, Column: 3}, }, - DocString: "test", Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/ast/function_declaration.go b/ast/function_declaration.go index c4bcc13744..b481cfe8cc 100644 --- a/ast/function_declaration.go +++ b/ast/function_declaration.go @@ -20,6 +20,7 @@ package ast import ( "encoding/json" + "github.com/turbolent/prettier" "github.com/onflow/cadence/common" diff --git a/ast/position.go b/ast/position.go index 8537158349..6d4b86f4b5 100644 --- a/ast/position.go +++ b/ast/position.go @@ -20,6 +20,7 @@ package ast import ( "fmt" + "github.com/onflow/cadence/common" ) @@ -206,9 +207,9 @@ func (r Range) EndPosition(common.MemoryGauge) Position { return r.EndPos } -func (e Range) Source(input []byte) []byte { - startOffset := e.StartPos.Offset - endOffset := e.EndPos.Offset + 1 +func (r Range) Source(input []byte) []byte { + startOffset := r.StartPos.Offset + endOffset := r.EndPos.Offset + 1 return input[startOffset:endOffset] } diff --git a/ast/statement.go b/ast/statement.go index 6bb36b19d2..48980eb31e 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -871,32 +871,19 @@ type SwitchCase struct { Comments } -func NewSwitchCase( - _ common.MemoryGauge, - expression Expression, - statements []Statement, - r Range, - comments Comments, -) *SwitchCase { - return &SwitchCase{ - Expression: expression, - Statements: statements, - Range: r, - Comments: comments, - } -} - func NewSwitchCase( gauge common.MemoryGauge, expression Expression, statements []Statement, astRange Range, + comments Comments, ) *SwitchCase { common.UseMemory(gauge, common.SwitchCaseMemoryUsage) return &SwitchCase{ Expression: expression, Statements: statements, Range: astRange, + Comments: comments, } } diff --git a/ast/transaction_declaration_test.go b/ast/transaction_declaration_test.go index c407759328..9381524649 100644 --- a/ast/transaction_declaration_test.go +++ b/ast/transaction_declaration_test.go @@ -60,7 +60,7 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { NewComment(nil, []byte("///test")), }, }, - Execute: nil, + Execute: nil, Range: Range{ StartPos: Position{Offset: 7, Line: 8, Column: 9}, EndPos: Position{Offset: 10, Line: 11, Column: 12}, diff --git a/bbq/compiler/desugar.go b/bbq/compiler/desugar.go index a1df108397..d9959d1cf4 100644 --- a/bbq/compiler/desugar.go +++ b/bbq/compiler/desugar.go @@ -217,7 +217,7 @@ func (d *Desugar) VisitFunctionDeclaration(declaration *ast.FunctionDeclaration, declaration.ReturnTypeAnnotation, modifiedFuncBlock, declaration.StartPos, - declaration.DocString, + declaration.Comments, ) d.elaboration.SetFunctionDeclarationFunctionType(modifiedDecl, functionType) @@ -297,6 +297,7 @@ func (d *Desugar) desugarFunctionBlock( d.memoryGauge, modifiedStatements, ast.NewRangeFromPositioned(d.memoryGauge, hasPosition), + ast.EmptyComments, ), nil, nil, @@ -329,7 +330,7 @@ func (d *Desugar) tempResultVariable( pos, nil, nil, - "", + ast.EmptyComments, ) d.elaboration.SetVariableDeclarationTypes( @@ -392,7 +393,7 @@ func (d *Desugar) declareResultVariable( pos, nil, nil, - "", + ast.EmptyComments, ) d.elaboration.SetVariableDeclarationTypes( @@ -416,6 +417,7 @@ func (d *Desugar) tempResultIdentifierExpr(pos ast.Position) *ast.IdentifierExpr tempResultVariableName, pos, ), + ast.EmptyComments, ) } @@ -717,6 +719,7 @@ func (d *Desugar) desugarCondition( functionName, startPos, ), + ast.EmptyComments, ), nil, []*ast.Argument{ @@ -748,9 +751,11 @@ func (d *Desugar) desugarCondition( d.memoryGauge, condition.Test, ), + ast.EmptyComments, ), nil, startPos, + ast.EmptyComments, ) return ifStmt @@ -798,6 +803,7 @@ func (d *Desugar) desugarCondition( declaredContract.GetIdentifier(), pos, ), + ast.EmptyComments, ), false, pos, @@ -821,6 +827,7 @@ func (d *Desugar) desugarCondition( d.memoryGauge, newEventConstructorInvocation, emitStmt.StartPos, + ast.EmptyComments, ) //Inject a static import so the compiler can link the functions. @@ -982,8 +989,8 @@ func (d *Desugar) VisitAttachmentDeclaration(declaration *ast.AttachmentDeclarat declaration.BaseType, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1104,8 +1111,8 @@ func (d *Desugar) VisitCompositeDeclaration(declaration *ast.CompositeDeclaratio declaration.Identifier, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1270,7 +1277,7 @@ func (d *Desugar) inheritedDefaultFunctions( ) funcReturnType := inheritedFuncType.ReturnTypeAnnotation.Type - returnStmt := ast.NewReturnStatement(d.memoryGauge, invocation, declRange) + returnStmt := ast.NewReturnStatement(d.memoryGauge, invocation, declRange, ast.EmptyComments) d.elaboration.SetReturnStatementTypes( returnStmt, sema.ReturnStatementTypes{ @@ -1302,12 +1309,13 @@ func (d *Desugar) inheritedDefaultFunctions( returnStmt, }, declRange, + ast.EmptyComments, ), nil, nil, ), inheritedFunc.StartPos, - inheritedFunc.DocString, + inheritedFunc.Comments, ) d.elaboration.SetFunctionDeclarationFunctionType(defaultFuncDelegator, inheritedFuncType) @@ -1377,6 +1385,7 @@ func (d *Desugar) interfaceDelegationMethodCall( param.Identifier, pos, ), + ast.EmptyComments, ), ) } else { @@ -1399,6 +1408,7 @@ func (d *Desugar) interfaceDelegationMethodCall( param.Identifier, pos, ), + ast.EmptyComments, ), ) } @@ -1424,12 +1434,12 @@ func (d *Desugar) interfaceDelegationMethodCall( d.memoryGauge, ast.NewIdentifierExpression( d.memoryGauge, - ast.NewIdentifier( d.memoryGauge, "self", pos, ), + ast.EmptyComments, ), false, pos, @@ -1520,6 +1530,7 @@ func (d *Desugar) addImport(location common.Location) { location, ast.EmptyRange, ast.EmptyPosition, + ast.EmptyComments, ) d.newImports = append( @@ -1558,8 +1569,8 @@ func (d *Desugar) VisitInterfaceDeclaration(declaration *ast.InterfaceDeclaratio declaration.Identifier, declaration.Conformances, ast.NewMembers(d.memoryGauge, desugaredMembers), - declaration.DocString, declaration.Range, + declaration.Comments, ) // Update elaboration. Type info is needed for later steps. @@ -1600,7 +1611,7 @@ func (d *Desugar) VisitTransactionDeclaration(transaction *ast.TransactionDeclar parameter.StartPos, nil, nil, - "", + ast.EmptyComments, ) varDeclarations = append(varDeclarations, variableDecl) @@ -1666,8 +1677,8 @@ func (d *Desugar) VisitTransactionDeclaration(transaction *ast.TransactionDeclar ), nil, ast.NewMembers(d.memoryGauge, members), - "", ast.EmptyRange, + ast.EmptyComments, ) compositeType := d.transactionCompositeType() @@ -1780,12 +1791,12 @@ var emptyInitializer = func() *ast.SpecialFunctionDeclaration { nil, ast.NewFunctionBlock( nil, - ast.NewBlock(nil, nil, ast.EmptyRange), + ast.NewBlock(nil, nil, ast.EmptyRange, ast.EmptyComments), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) return ast.NewSpecialFunctionDeclaration( @@ -1843,6 +1854,7 @@ func newEnumInitializer( ), nil, ast.EmptyPosition, + ast.EmptyComments, ), } @@ -1855,6 +1867,7 @@ func newEnumInitializer( sema.SelfIdentifier, ast.EmptyPosition, ), + ast.EmptyComments, ), false, ast.EmptyPosition, @@ -1889,6 +1902,7 @@ func newEnumInitializer( ast.NewIdentifierExpression( gauge, rawValueIdentifier, + ast.EmptyComments, ), ) @@ -1912,7 +1926,7 @@ func newEnumInitializer( ast.EmptyPosition, ), nil, - ast.NewParameterList(gauge, parameters, ast.EmptyRange), + ast.NewParameterList(gauge, parameters, ast.EmptyRange, ast.EmptyComments), nil, ast.NewFunctionBlock( gauge, @@ -1922,12 +1936,13 @@ func newEnumInitializer( assignmentStatement, }, ast.EmptyRange, + ast.EmptyComments, ), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) return ast.NewSpecialFunctionDeclaration( @@ -1993,6 +2008,7 @@ func newEnumLookup( ), nil, ast.EmptyPosition, + ast.EmptyComments, ), } @@ -2011,6 +2027,7 @@ func newEnumLookup( big.NewInt(int64(index)), 10, ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetIntegerExpressionType( integerExpression, @@ -2022,6 +2039,7 @@ func newEnumLookup( ast.NewIdentifierExpression( gauge, typeIdentifier, + ast.EmptyComments, ), false, ast.EmptyPosition, @@ -2043,6 +2061,7 @@ func newEnumLookup( gauge, memberExpression, ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetReturnStatementTypes( @@ -2070,6 +2089,7 @@ func newEnumLookup( gauge, ast.NewNilExpression(gauge, ast.EmptyPosition), ast.EmptyRange, + ast.EmptyComments, ) elaboration.SetReturnStatementTypes( @@ -2091,9 +2111,10 @@ func newEnumLookup( switchStatement := ast.NewSwitchStatement( gauge, - ast.NewIdentifierExpression(gauge, rawValueIdentifier), + ast.NewIdentifierExpression(gauge, rawValueIdentifier, ast.EmptyComments), switchCases, ast.EmptyRange, + ast.EmptyComments, ) return ast.NewFunctionDeclaration( @@ -2104,7 +2125,7 @@ func newEnumLookup( false, typeIdentifier, nil, - ast.NewParameterList(gauge, parameters, ast.EmptyRange), + ast.NewParameterList(gauge, parameters, ast.EmptyRange, ast.EmptyComments), nil, ast.NewFunctionBlock( gauge, @@ -2114,12 +2135,13 @@ func newEnumLookup( switchStatement, }, ast.EmptyRange, + ast.EmptyComments, ), nil, nil, ), ast.EmptyPosition, - "", + ast.EmptyComments, ) } @@ -2156,6 +2178,7 @@ func simpleFunctionDeclaration( memoryGauge, parameters, astRange, + ast.EmptyComments, ) } @@ -2179,12 +2202,13 @@ func simpleFunctionDeclaration( memoryGauge, statements, astRange, + ast.EmptyComments, ), nil, nil, ), startPos, - "", + ast.EmptyComments, ) } @@ -2260,6 +2284,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( enclosingType.QualifiedString(), startPos, ), + ast.EmptyComments, ), false, startPos, @@ -2375,6 +2400,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( commons.CollectEventsParamName, startPos, ), + ast.EmptyComments, ), nil, eventConstructorInvocations, @@ -2422,6 +2448,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( ), nil, ast.EmptyPosition, + ast.EmptyComments, ) eventEmittingFunction := ast.NewFunctionDeclaration( @@ -2440,6 +2467,7 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( d.memoryGauge, []*ast.Parameter{parameter}, astRange, + ast.EmptyComments, ), nil, ast.NewFunctionBlock( @@ -2448,12 +2476,13 @@ func (d *Desugar) generateResourceDestroyedEventsGetterFunction( d.memoryGauge, []ast.Statement{expressionStmt}, astRange, + ast.EmptyComments, ), nil, nil, ), startPos, - "", + ast.EmptyComments, ) d.elaboration.SetFunctionDeclarationFunctionType(eventEmittingFunction, eventEmittingFunctionType) @@ -2489,6 +2518,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( sema.SelfIdentifier, pos, ), + ast.EmptyComments, ), false, pos, @@ -2536,6 +2566,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( parameterName, pos, ), + ast.EmptyComments, ), ) @@ -2554,6 +2585,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( d.memoryGauge, statements, ast.NewRangeFromPositioned(d.memoryGauge, initializer), + ast.EmptyComments, ), nil, nil, @@ -2571,7 +2603,7 @@ func (d *Desugar) desugarDefaultDestroyEventInitializer( initializer.ReturnTypeAnnotation, modifiedFuncBlock, initializer.StartPos, - initializer.DocString, + initializer.Comments, ) // Desugared function's type is same as the original function's type. diff --git a/old_parser/comment.go b/old_parser/comment.go deleted file mode 100644 index 955c66cc89..0000000000 --- a/old_parser/comment.go +++ /dev/null @@ -1 +0,0 @@ -// TODO(merge): Remove \ No newline at end of file diff --git a/old_parser/expression_test.go b/old_parser/expression_test.go index 153c602895..5700783e2a 100644 --- a/old_parser/expression_test.go +++ b/old_parser/expression_test.go @@ -33,7 +33,6 @@ import ( "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" - "github.com/onflow/cadence/old_parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) diff --git a/old_parser/function.go b/old_parser/function.go index 42c56c0125..926d56d348 100644 --- a/old_parser/function.go +++ b/old_parser/function.go @@ -343,9 +343,7 @@ func parseFunctionDeclaration( returnTypeAnnotation, functionBlock, startPos, - ast.Comments{ - Leading: startToken.Leading, - }, + ast.EmptyComments, ), nil } diff --git a/old_parser/statement.go b/old_parser/statement.go index 2cd588520b..0e548cbf22 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -194,9 +194,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State returnTypeAnnotation, functionBlock, startToken.StartPos, - ast.Comments{ - Leading: startToken.Comments.PackToList(), - }, + ast.EmptyComments, ), nil } else { parameterList, returnTypeAnnotation, functionBlock, err := diff --git a/parser/comment.go b/parser/comment.go deleted file mode 100644 index 955c66cc89..0000000000 --- a/parser/comment.go +++ /dev/null @@ -1 +0,0 @@ -// TODO(merge): Remove \ No newline at end of file diff --git a/parser/comment_test.go b/parser/comment_test.go index ea99409e42..54ebcbe974 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -22,10 +22,10 @@ import ( "math/big" "testing" + "github.com/onflow/cadence/errors" "github.com/stretchr/testify/require" "github.com/onflow/cadence/ast" - "github.com/onflow/cadence/errors" "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) @@ -34,13 +34,6 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() - t.Run("empty", func(t *testing.T) { - t.Parallel() - - _, errs := testParseExpression(`/**/ true`) - require.Empty(t, errs) - }) - t.Run("nested", func(t *testing.T) { t.Parallel() @@ -83,6 +76,8 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() + // TODO(preserve-comments): Extracting comments from operator tokens is a bit difficult, + // so let's handle this later as it seems a pretty edge case location to add comments. result, errs := testParseExpression(" 1/*test foo*/+/* bar */ 2 ") require.Empty(t, errs) @@ -97,6 +92,11 @@ func TestParseBlockComment(t *testing.T) { StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/*test foo*/")), + }, + }, }, Right: &ast.IntegerExpression{ PositiveLiteral: []byte("2"), @@ -202,13 +202,7 @@ func TestParseBlockComment(t *testing.T) { tokens := &testTokenStream{ tokens: []lexer.Token{ - { - Type: lexer.TokenBlockCommentStart, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Offset: 0, Column: 0}, - EndPos: ast.Position{Line: 1, Offset: 1, Column: 1}, - }, - }, + // TODO(merge): is this correct? { Type: lexer.TokenIdentifier, Range: ast.Range{ @@ -221,6 +215,7 @@ func TestParseBlockComment(t *testing.T) { input: []byte(`/*foo`), } + // TODO(merge): move and emit UnexpectedTokenInBlockCommentError from lexer _, errs := ParseTokenStream( nil, tokens, @@ -245,3 +240,144 @@ func TestParseBlockComment(t *testing.T) { ) }) } + +func TestParseWhileStatementComment(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// before if +if true { + // noop +} // after if +// before else-if +else if true { + // noop +} +// before second else-if +else if true { + // noop +} /* after else-if */ else { + // noop +} // after else +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, + EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, + EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after if")), + }, + }, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, + EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, + EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + }, + }, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, + EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, + EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("/* after else-if */")), + }, + }, + }, + Else: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after else")), + }, + }, + }, + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before second else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before if")), + }, + }, + }, + }, + result, + ) +} diff --git a/parser/declaration.go b/parser/declaration.go index e45d4d6973..7fa7e6c5b7 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -204,7 +204,7 @@ func parseDeclaration(p *parser) (ast.Declaration, error) { if staticToken != nil { p.reportSyntaxError("invalid second `static` modifier") } - if nativeModifierEnabled && nativePos != nil { + if nativeModifierEnabled && nativeToken != nil { p.reportSyntaxError("invalid `static` modifier after `native` modifier") } tok := p.current @@ -712,7 +712,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { // Parse optional alias alias := parseOptionalImportAlias(p) - p.skipSpaceAndComments() + p.skipSpace() imports = append( imports, @@ -755,7 +755,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { // Parse optional alias alias := parseOptionalImportAlias(p) - p.skipSpaceAndComments() + p.skipSpace() switch p.current.Type { case lexer.TokenComma: @@ -1678,8 +1678,8 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { ) case KeywordCase: - rejectNonAccessModifiers(p, staticPos, nativePos, purityPos, common.DeclarationKindEnumCase) - return parseEnumCase(p, access, accessPos, docString) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEnumCase) + return parseEnumCase(p, access, accessToken) case KeywordFun: return parseFunctionDeclaration( diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 79dbef0dc0..9b350d7664 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -3308,7 +3308,6 @@ import "foo" /* After foo */`) AssertEqualWithDiff(t, []ast.Declaration{ &ast.ImportDeclaration{ - Identifiers: nil, Location: common.StringLocation("foo"), LocationPos: ast.Position{Line: 3, Column: 7, Offset: 22}, Range: ast.Range{ @@ -4621,9 +4620,7 @@ func TestParseField(t *testing.T) { return Parse( nil, []byte(input), - func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration(p) - }, + parseMemberOrNestedDeclaration, config, ) } @@ -5401,7 +5398,7 @@ func TestParseCompositeDeclaration(t *testing.T) { Column: 19, }, }, - AccessPos: ast.Position{ + AccessEndPos: ast.Position{ Offset: 172, Line: 9, Column: 18, @@ -5521,7 +5518,7 @@ func TestParseCompositeDeclaration(t *testing.T) { Column: 26, }, }, - AccessPos: ast.Position{ + AccessEndPos: ast.Position{ Offset: 290, Line: 14, Column: 25, @@ -10802,9 +10799,7 @@ func TestParseNestedPragma(t *testing.T) { return Parse( nil, []byte(input), - func(p *parser) (ast.Declaration, error) { - return parseMemberOrNestedDeclaration(p) - }, + parseMemberOrNestedDeclaration, config, ) } diff --git a/parser/errors.go b/parser/errors.go index d18cda2aba..19c66a622c 100644 --- a/parser/errors.go +++ b/parser/errors.go @@ -3975,11 +3975,9 @@ func (e *MissingCommentEndError) EndPosition(_ common.MemoryGauge) ast.Position return e.Pos } +// TODO(merge): Move and emit from lexer func (e *MissingCommentEndError) Error() string { - return fmt.Sprintf( - "missing comment end (%#q)", - lexer.TokenBlockCommentEnd, - ) + return "missing comment end (`*/`)" } func (*MissingCommentEndError) SecondaryError() string { diff --git a/parser/expression_test.go b/parser/expression_test.go index f5cefb09b4..bf27863375 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -28,6 +28,7 @@ import ( "testing" "testing/quick" + "github.com/onflow/cadence/parser/lexer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -2542,146 +2543,6 @@ func TestParseMemberExpression(t *testing.T) { }) } -// TODO(merge): Resolve with duplicate declaration -func TestParseBlockComment(t *testing.T) { - - t.Parallel() - - t.Run("nested", func(t *testing.T) { - - t.Parallel() - - result, errs := testParseExpression(" /* test foo/* bar */ asd*/ true") - require.Empty(t, errs) - - AssertEqualWithDiff(t, - &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 30, Offset: 30}, - EndPos: ast.Position{Line: 1, Column: 33, Offset: 33}, - }, - }, - result, - ) - }) - - t.Run("two comments", func(t *testing.T) { - - t.Parallel() - - result, errs := testParseExpression(" /*test foo*/ /* bar */ true") - require.Empty(t, errs) - - AssertEqualWithDiff(t, - &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 26, Offset: 26}, - EndPos: ast.Position{Line: 1, Column: 29, Offset: 29}, - }, - }, - result, - ) - }) - - t.Run("in infix", func(t *testing.T) { - - t.Parallel() - - // TODO(preserve-comments): Extracting comments from operator tokens is a bit difficult, - // so let's handle this later as it seems a pretty edge case location to add comments. - result, errs := testParseExpression(" 1/*test foo*/+/* bar */ 2 ") - require.Empty(t, errs) - - AssertEqualWithDiff(t, - &ast.BinaryExpression{ - Operation: ast.OperationPlus, - Left: &ast.IntegerExpression{ - PositiveLiteral: []byte("1"), - Value: big.NewInt(1), - Base: 10, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - EndPos: ast.Position{Line: 1, Column: 1, Offset: 1}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("/*test foo*/")), - }, - }, - }, - Right: &ast.IntegerExpression{ - PositiveLiteral: []byte("2"), - Value: big.NewInt(2), - Base: 10, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - EndPos: ast.Position{Line: 1, Column: 27, Offset: 27}, - }, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/* bar */")), - }, - }, - }, - }, - result, - ) - }) - - t.Run("nested, extra closing", func(t *testing.T) { - - t.Parallel() - - _, errs := testParseExpression(" /* test foo/* bar */ asd*/ true */ bar") - AssertEqualWithDiff(t, - []error{ - // `true */ bar` is parsed as infix operation of path - &SyntaxError{ - Message: "expected token identifier", - Pos: ast.Position{ - Offset: 37, - Line: 1, - Column: 37, - }, - }, - }, - errs, - ) - }) - - t.Run("nested, missing closing", func(t *testing.T) { - - t.Parallel() - - _, errs := testParseExpression(" /* test foo/* bar */ asd true ") - AssertEqualWithDiff(t, - []error{ - // `true */ bar` is parsed as infix operation of path - &SyntaxError{ - Message: "missing comment end '*/'", - Pos: ast.Position{ - Offset: 33, - Line: 1, - Column: 33, - }, - }, - &SyntaxError{ - Message: "unexpected end of program", - Pos: ast.Position{ - Offset: 33, - Line: 1, - Column: 33, - }, - }, - }, - errs, - ) - }) - -} - func TestParseMulInfixExpression(t *testing.T) { t.Parallel() diff --git a/parser/function.go b/parser/function.go index aac451b3e7..522c3ddad6 100644 --- a/parser/function.go +++ b/parser/function.go @@ -175,9 +175,10 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro p.nextSemanticToken() } - if !p.current.Is(lexer.TokenColon) { + colonToken := p.current + if !colonToken.Is(lexer.TokenColon) { return nil, &MissingColonAfterParameterNameError{ - GotToken: p.current, + GotToken: colonToken, } } diff --git a/parser/statement.go b/parser/statement.go index 8fdba0f8b5..910da03d02 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -280,7 +280,6 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: tok := p.current endToken = &tok - break default: if !sawNewLine { expression, err = parseExpression(p, lowestBindingPower) @@ -385,7 +384,6 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { // Skip the `else` keyword p.nextSemanticToken() - // The parser ignores the `else` token, // so to preserve potential comments associated with else token, // we attach the comments to the (next) `if` token. @@ -543,7 +541,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { expression, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), + Leading: startToken.Comments.PackToList(), // TODO(preserve-comments): handle inToken=nil Trailing: inToken.Comments.PackToList(), }, diff --git a/parser/statement_test.go b/parser/statement_test.go index b146ab1e98..cbd76a9eff 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -682,148 +682,6 @@ func TestParseIfStatement(t *testing.T) { }) } -func TestParseWhileStatement(t *testing.T) { - - t.Parallel() - - result, errs := testParseStatements(` -// before if -if true { - // noop -} // after if -// before else-if -else if true { - // noop -} -// before second else-if -else if true { - // noop -} /* after else-if */ else { - // noop -} // after else -`) - require.Empty(t, errs) - - AssertEqualWithDiff(t, - []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, - EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, - EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("// after if")), - }, - }, - }, - Else: &ast.Block{ - Statements: []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, - EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, - EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - }, - }, - }, - Else: &ast.Block{ - Statements: []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, - EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, - EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("/* after else-if */")), - }, - }, - }, - Else: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("// after else")), - }, - }, - }, - StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before second else-if")), - }, - }, - }, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - }, - StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before else-if")), - }, - }, - }, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - }, - StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before if")), - }, - }, - }, - }, - result, - ) - }) -} - func TestParseAssignmentStatement(t *testing.T) { t.Parallel() From 8260761aa30fdf030b53b04240e8fa7ede570861 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 18 Nov 2025 21:03:46 +0100 Subject: [PATCH 60/84] replaces all usages of `ast.Comments{}` with `ast.EmptyComments` --- interpreter/interpreter_statement.go | 2 +- old_parser/declaration.go | 22 +++++++++++----------- old_parser/declaration_test.go | 6 +++--- old_parser/expression.go | 4 ++-- old_parser/function.go | 4 ++-- old_parser/statement.go | 24 ++++++++++++------------ old_parser/transaction.go | 4 ++-- old_parser/type.go | 2 +- parser/comment_test.go | 3 ++- parser/declaration.go | 2 +- parser/expression_test.go | 3 ++- parser/statement.go | 4 ++-- sema/before_extractor.go | 2 +- sema/check_switch.go | 2 +- 14 files changed, 43 insertions(+), 41 deletions(-) diff --git a/interpreter/interpreter_statement.go b/interpreter/interpreter_statement.go index 347f1456d7..3a14a542cc 100644 --- a/interpreter/interpreter_statement.go +++ b/interpreter/interpreter_statement.go @@ -182,7 +182,7 @@ func (interpreter *Interpreter) VisitSwitchStatement(switchStatement *ast.Switch interpreter, switchCase.Statements, ast.EmptyRange, - ast.Comments{}, + ast.EmptyComments, ) result := interpreter.visitBlock(block) diff --git a/old_parser/declaration.go b/old_parser/declaration.go index dccbebcef0..528538ee51 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -415,7 +415,7 @@ func parseVariableDeclaration( startPos, secondTransfer, secondValue, - ast.Comments{}, + ast.EmptyComments, ) castingExpression, leftIsCasting := value.(*ast.CastingExpression) @@ -710,7 +710,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { endPos, ), locationPos, - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -817,7 +817,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - ast.Comments{}, + ast.EmptyComments, ), ) @@ -840,7 +840,7 @@ func parseEventDeclaration( startPos, parameterList.EndPos, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -931,7 +931,7 @@ func parseFieldWithVariableKind( startPos, typeAnnotation.EndPosition(p.memoryGauge), ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -1065,7 +1065,7 @@ func parseCompositeOrInterfaceDeclaration( []*ast.NominalType{}, members, declarationRange, - ast.Comments{}, + ast.EmptyComments, ), nil } else { return ast.NewCompositeDeclaration( @@ -1076,7 +1076,7 @@ func parseCompositeOrInterfaceDeclaration( conformances, members, declarationRange, - ast.Comments{}, + ast.EmptyComments, ), nil } } @@ -1168,7 +1168,7 @@ func parseAttachmentDeclaration( conformances, members, declarationRange, - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -1486,7 +1486,7 @@ func parseFieldDeclarationWithoutVariableKind( startPos, typeAnnotation.EndPosition(p.memoryGauge), ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -1552,7 +1552,7 @@ func parseSpecialFunctionDeclaration( nil, functionBlock, startPos, - ast.Comments{}, + ast.EmptyComments, ), ), nil } @@ -1589,7 +1589,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - ast.Comments{}, + ast.EmptyComments, startPos, ), nil } diff --git a/old_parser/declaration_test.go b/old_parser/declaration_test.go index d2516b5194..44bc98c3ba 100644 --- a/old_parser/declaration_test.go +++ b/old_parser/declaration_test.go @@ -7062,7 +7062,7 @@ func TestParseMemberDocStrings(t *testing.T) { Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{}, + Comments: ast.EmptyComments, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7080,7 +7080,7 @@ func TestParseMemberDocStrings(t *testing.T) { Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{}, + Comments: ast.EmptyComments, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7098,7 +7098,7 @@ func TestParseMemberDocStrings(t *testing.T) { Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{}, + Comments: ast.EmptyComments, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/old_parser/expression.go b/old_parser/expression.go index 5d340b8e17..ee13be3816 100644 --- a/old_parser/expression.go +++ b/old_parser/expression.go @@ -821,7 +821,7 @@ func defineIdentifierExpression() { return ast.NewIdentifierExpression( p.memoryGauge, p.tokenToIdentifier(token), - ast.Comments{}, + ast.EmptyComments, ), nil } }, @@ -1772,7 +1772,7 @@ func parseIntegerLiteral(p *parser, literal, text []byte, kind common.IntegerLit value = new(big.Int) } - return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange, ast.Comments{}) + return ast.NewIntegerExpression(p.memoryGauge, literal, value, base, tokenRange, ast.EmptyComments) } func parseFixedPointPart(gauge common.MemoryGauge, part string) (integer *big.Int, scale uint) { diff --git a/old_parser/function.go b/old_parser/function.go index 926d56d348..fc7256711f 100644 --- a/old_parser/function.go +++ b/old_parser/function.go @@ -108,7 +108,7 @@ func parseParameterList(p *parser) (*ast.ParameterList, error) { startPos, endPos, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -168,7 +168,7 @@ func parseParameter(p *parser) (*ast.Parameter, error) { typeAnnotation, nil, startPos, - ast.Comments{}, + ast.EmptyComments, ), nil } diff --git a/old_parser/statement.go b/old_parser/statement.go index 0e548cbf22..7f2ea66cd9 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -250,7 +250,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { tokenRange.StartPos, endPosition, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -258,14 +258,14 @@ func parseBreakStatement(p *parser) *ast.BreakStatement { tokenRange := p.current.Range p.next() - return ast.NewBreakStatement(p.memoryGauge, tokenRange, ast.Comments{}) + return ast.NewBreakStatement(p.memoryGauge, tokenRange, ast.EmptyComments) } func parseContinueStatement(p *parser) *ast.ContinueStatement { tokenRange := p.current.Range p.next() - return ast.NewContinueStatement(p.memoryGauge, tokenRange, ast.Comments{}) + return ast.NewContinueStatement(p.memoryGauge, tokenRange, ast.EmptyComments) } func parseIfStatement(p *parser) (*ast.IfStatement, error) { @@ -337,7 +337,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { thenBlock, elseBlock, startPos, - ast.Comments{}, + ast.EmptyComments, ) if variableDeclaration != nil { @@ -361,7 +361,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), - ast.Comments{}, + ast.EmptyComments, ) result = outer } @@ -384,7 +384,7 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { return nil, err } - return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos, ast.Comments{}), nil + return ast.NewWhileStatement(p.memoryGauge, expression, block, startPos, ast.EmptyComments), nil } func parseForStatement(p *parser) (*ast.ForStatement, error) { @@ -450,7 +450,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { block, expression, startPos, - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -480,7 +480,7 @@ func parseBlock(p *parser) (*ast.Block, error) { startToken.StartPos, endToken.EndPos, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -536,7 +536,7 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { startToken.StartPos, endToken.EndPos, ), - ast.Comments{}, + ast.EmptyComments, ), preConditions, postConditions, @@ -618,7 +618,7 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { return nil, err } - return ast.NewEmitStatement(p.memoryGauge, invocation, startPos, ast.Comments{}), nil + return ast.NewEmitStatement(p.memoryGauge, invocation, startPos, ast.EmptyComments), nil } func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { @@ -657,7 +657,7 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { startPos, endToken.EndPos, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -827,6 +827,6 @@ func parseRemoveStatement( attachmentNominalType, attached, startPos, - ast.Comments{}, + ast.EmptyComments, ), nil } diff --git a/old_parser/transaction.go b/old_parser/transaction.go index 06c66e6a19..03f37db257 100644 --- a/old_parser/transaction.go +++ b/old_parser/transaction.go @@ -203,7 +203,7 @@ func parseTransactionDeclaration(p *parser, docString string) (*ast.TransactionD startPos, endPos, ), - ast.Comments{}, + ast.EmptyComments, ), nil } @@ -282,7 +282,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) nil, ), identifier.Pos, - ast.Comments{}, + ast.EmptyComments, ), ), nil } diff --git a/old_parser/type.go b/old_parser/type.go index ce2e201132..d04f8cbd88 100644 --- a/old_parser/type.go +++ b/old_parser/type.go @@ -883,7 +883,7 @@ func parseNominalTypeInvocationRemainder(p *parser) (*ast.InvocationExpression, var invokedExpression ast.Expression = ast.NewIdentifierExpression( p.memoryGauge, ty.Identifier, - ast.Comments{}, + ast.EmptyComments, ) for _, nestedIdentifier := range ty.NestedIdentifiers { diff --git a/parser/comment_test.go b/parser/comment_test.go index 54ebcbe974..90b2d96307 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -22,9 +22,10 @@ import ( "math/big" "testing" - "github.com/onflow/cadence/errors" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/errors" + "github.com/onflow/cadence/ast" "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" diff --git a/parser/declaration.go b/parser/declaration.go index 7fa7e6c5b7..8e7b29aec4 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -963,7 +963,7 @@ func parseEventDeclaration( nil, nil, parameterList.StartPos, - ast.Comments{}, + ast.EmptyComments, ), ) diff --git a/parser/expression_test.go b/parser/expression_test.go index bf27863375..3ddad824b2 100644 --- a/parser/expression_test.go +++ b/parser/expression_test.go @@ -28,10 +28,11 @@ import ( "testing" "testing/quick" - "github.com/onflow/cadence/parser/lexer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/onflow/cadence/parser/lexer" + "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" diff --git a/parser/statement.go b/parser/statement.go index 910da03d02..e061a4e0cf 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -291,7 +291,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { } } - comments := ast.Comments{} + comments := ast.EmptyComments if endToken == nil { comments = startToken.Comments } else { @@ -443,7 +443,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { p.memoryGauge, []ast.Statement{result}, ast.NewRangeFromPositioned(p.memoryGauge, result), - ast.Comments{}, + ast.EmptyComments, ) result = outer } diff --git a/sema/before_extractor.go b/sema/before_extractor.go index 034aa56eb0..041e7b9e05 100644 --- a/sema/before_extractor.go +++ b/sema/before_extractor.go @@ -76,7 +76,7 @@ func (e *BeforeExtractor) ExtractInvocation( ast.EmptyPosition, ) - newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier, ast.Comments{}) + newExpression := ast.NewIdentifierExpression(e.memoryGauge, newIdentifier, ast.EmptyComments) extractedExpressions = append(extractedExpressions, ast.ExtractedExpression{ diff --git a/sema/check_switch.go b/sema/check_switch.go index 0a73f785ca..8f0e39adcf 100644 --- a/sema/check_switch.go +++ b/sema/check_switch.go @@ -185,7 +185,7 @@ func (checker *Checker) checkSwitchCaseStatements(switchCase *ast.SwitchCase) { switchCase.Statements[0].StartPosition(), switchCase.EndPos, ), - ast.Comments{}, + ast.EmptyComments, ) checker.checkBlock(block) } From 01ba32b478a51d5ed0fd0fc191a0922c077d2e8b Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Wed, 19 Nov 2025 21:01:50 +0100 Subject: [PATCH 61/84] fix invalid arguments --- parser/declaration.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/parser/declaration.go b/parser/declaration.go index 8e7b29aec4..d6cefedb8f 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -1694,27 +1694,27 @@ func parseMemberOrNestedDeclaration(p *parser) (ast.Declaration, error) { ) case KeywordEvent: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindEvent) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEvent) return parseEventDeclaration(p, access, accessToken) case KeywordStruct: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindStructure) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindStructure) return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordResource: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindResource) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindResource) return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordContract: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindContract) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindContract) return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordEntitlement: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindEntitlement) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEntitlement) return parseEntitlementOrMappingDeclaration(p, access, accessToken) case KeywordEnum: - rejectNonAccessModifiers(p, accessToken, accessToken, accessToken, common.DeclarationKindEnum) + rejectNonAccessModifiers(p, staticToken, nativeToken, purityToken, common.DeclarationKindEnum) return parseCompositeOrInterfaceDeclaration(p, access, accessToken) case KeywordAttachment: From 88a6550e47b1c8667ded5cb69c2a583c27306ad6 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Wed, 19 Nov 2025 21:52:26 +0100 Subject: [PATCH 62/84] fix json format assertions to include comments field --- ast/attachment_test.go | 3 +++ ast/block_test.go | 1 + ast/expression_test.go | 7 +++++++ ast/statement_test.go | 6 ++++++ 4 files changed, 17 insertions(+) diff --git a/ast/attachment_test.go b/ast/attachment_test.go index 6f8a7f3614..9442f2532a 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -302,6 +302,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "EndPos": {"Offset": 3, "Line": 2, "Column": 5}, "Base": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foo", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -314,6 +315,7 @@ func TestAttachExpressionMarshallJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "bar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -473,6 +475,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { "EndPos": {"Offset": 3, "Line": 2, "Column": 5}, "Value": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "baz", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/block_test.go b/ast/block_test.go index 7caf615bca..7d94c74537 100644 --- a/ast/block_test.go +++ b/ast/block_test.go @@ -360,6 +360,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 31, "Line": 32, "Column": 33}, diff --git a/ast/expression_test.go b/ast/expression_test.go index 8ddd609c82..1139f653d4 100644 --- a/ast/expression_test.go +++ b/ast/expression_test.go @@ -929,6 +929,7 @@ func TestIdentifierExpression_MarshalJSON(t *testing.T) { ` { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -2761,6 +2762,7 @@ func TestDestroyExpression_MarshalJSON(t *testing.T) { "Type": "DestroyExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -3005,6 +3007,7 @@ func TestForceExpression_MarshalJSON(t *testing.T) { "Type": "ForceExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -3834,6 +3837,7 @@ func TestInvocationExpression_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -4311,6 +4315,7 @@ func TestCastingExpression_MarshalJSON(t *testing.T) { "Type": "CastingExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -4709,6 +4714,7 @@ func TestCreateExpression_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -4865,6 +4871,7 @@ func TestReferenceExpression_MarshalJSON(t *testing.T) { "Type": "ReferenceExpression", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/statement_test.go b/ast/statement_test.go index ebbd681b17..e1a61d244a 100644 --- a/ast/statement_test.go +++ b/ast/statement_test.go @@ -1025,6 +1025,7 @@ func TestAssignmentStatement_MarshalJSON(t *testing.T) { "Type": "AssignmentStatement", "Target": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1185,6 +1186,7 @@ func TestSwapStatement_MarshalJSON(t *testing.T) { "Type": "SwapStatement", "Left": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1348,6 +1350,7 @@ func TestEmitStatement_MarshalJSON(t *testing.T) { "Type": "InvocationExpression", "InvokedExpression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1548,6 +1551,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "Type": "SwitchStatement", "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "foo", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1572,6 +1576,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "EndPos": {"Offset": 12, "Line": 11, "Column": 14}, "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "bar", "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, @@ -1595,6 +1600,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "EndPos": {"Offset": 21, "Line": 20, "Column": 23}, "Expression": { "Type": "IdentifierExpression", + "Comments": {}, "Identifier": { "Identifier": "baz", "StartPos": {"Offset": 19, "Line": 20, "Column": 21}, From 62cb4a87c0705782e5fe30d852392ab687611128 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Fri, 21 Nov 2025 14:39:46 +0100 Subject: [PATCH 63/84] revert all comment tracking changes in the old parser, tests passing --- ast/expression.go | 1 - old_parser/comment.go | 64 +++++++++++++++++++++++++++++++++ old_parser/declaration.go | 3 +- old_parser/declaration_test.go | 43 +++------------------- old_parser/expression_test.go | 66 ++++++++++++++++++++++++++++++++++ old_parser/function.go | 4 +-- old_parser/parser.go | 43 +++++++++++++++++++++- old_parser/statement.go | 6 ++-- 8 files changed, 182 insertions(+), 48 deletions(-) create mode 100644 old_parser/comment.go diff --git a/ast/expression.go b/ast/expression.go index 3296333f3a..7ff3dc5b40 100644 --- a/ast/expression.go +++ b/ast/expression.go @@ -47,7 +47,6 @@ type Expression interface { type BoolExpression struct { Value bool Range - Comments } var _ Element = &BoolExpression{} diff --git a/old_parser/comment.go b/old_parser/comment.go new file mode 100644 index 0000000000..b2cf999f5b --- /dev/null +++ b/old_parser/comment.go @@ -0,0 +1,64 @@ +/* + * Cadence - The resource-oriented smart contract programming language + * + * Copyright Flow Foundation + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package old_parser + +import ( + "github.com/onflow/cadence/old_parser/lexer" +) + +func (p *parser) parseBlockComment() (endToken lexer.Token, ok bool) { + var depth int + + for { + switch p.current.Type { + case lexer.TokenBlockCommentStart: + p.next() + depth++ + + case lexer.TokenBlockCommentContent: + p.next() + + case lexer.TokenBlockCommentEnd: + endToken = p.current + // Skip the comment end (`*/`) + p.next() + ok = true + depth-- + if depth == 0 { + return + } + + case lexer.TokenEOF: + p.reportSyntaxError( + "missing comment end %s", + lexer.TokenBlockCommentEnd, + ) + ok = false + return + + default: + p.reportSyntaxError( + "unexpected token %s in block comment", + p.current.Type, + ) + ok = false + return + } + } +} diff --git a/old_parser/declaration.go b/old_parser/declaration.go index 528538ee51..c4ca0c5d82 100644 --- a/old_parser/declaration.go +++ b/old_parser/declaration.go @@ -100,7 +100,7 @@ func parseDeclaration(p *parser, docString string) (ast.Declaration, error) { accessPos, staticPos, nativePos, - "", + docString, ) case keywordImport: @@ -1498,7 +1498,6 @@ func parseSpecialFunctionDeclaration( staticPos *ast.Position, nativePos *ast.Position, identifier ast.Identifier, - // Comments can be safely ignored docString string, ) (*ast.SpecialFunctionDeclaration, error) { diff --git a/old_parser/declaration_test.go b/old_parser/declaration_test.go index 44bc98c3ba..06eea2fbc5 100644 --- a/old_parser/declaration_test.go +++ b/old_parser/declaration_test.go @@ -820,9 +820,6 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - Comments: ast.Comments{ - Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Test"))}, - }, StartPos: ast.Position{Line: 2, Column: 0, Offset: 9}, }, }, @@ -860,12 +857,6 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/// First line")), - ast.NewComment(nil, []byte("/// Second line")), - }, - }, StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, @@ -903,11 +894,6 @@ func TestParseFunctionDeclaration(t *testing.T) { }, }, }, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/** Cool dogs.\n\n Cool cats!! */")), - }, - }, StartPos: ast.Position{Line: 7, Column: 0, Offset: 39}, }, }, @@ -6891,6 +6877,7 @@ func TestParseNestedPragma(t *testing.T) { } +// Doc strings are not tracked anymore in the old parser, and can be safely ignored. func TestParseMemberDocStrings(t *testing.T) { t.Parallel() @@ -6928,11 +6915,6 @@ func TestParseMemberDocStrings(t *testing.T) { []ast.Declaration{ &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/// noReturnNoBlock")), - }, - }, Identifier: ast.Identifier{ Identifier: "noReturnNoBlock", Pos: ast.Position{Offset: 78, Line: 5, Column: 18}, @@ -6947,11 +6929,6 @@ func TestParseMemberDocStrings(t *testing.T) { }, &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/// returnNoBlock")), - }, - }, Identifier: ast.Identifier{ Identifier: "returnNoBlock", Pos: ast.Position{Offset: 147, Line: 8, Column: 18}, @@ -6976,11 +6953,6 @@ func TestParseMemberDocStrings(t *testing.T) { }, &ast.FunctionDeclaration{ Access: ast.AccessNotSpecified, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/// returnAndBlock")), - }, - }, Identifier: ast.Identifier{ Identifier: "returnAndBlock", Pos: ast.Position{Offset: 220, Line: 11, Column: 18}, @@ -7007,10 +6979,6 @@ func TestParseMemberDocStrings(t *testing.T) { StartPos: ast.Position{Offset: 245, Line: 11, Column: 43}, EndPos: ast.Position{Offset: 246, Line: 11, Column: 44}, }, - Comments: ast.Comments{ - Leading: []*ast.Comment{}, - Trailing: []*ast.Comment{}, - }, }, }, StartPos: ast.Position{Offset: 216, Line: 11, Column: 14}, @@ -7061,8 +7029,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindUnknown, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.EmptyComments, + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "unknown", Pos: ast.Position{Offset: 66, Line: 5, Column: 14}, @@ -7079,8 +7046,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindInitializer, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.EmptyComments, + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "init", Pos: ast.Position{Offset: 121, Line: 8, Column: 14}, @@ -7097,8 +7063,7 @@ func TestParseMemberDocStrings(t *testing.T) { &ast.SpecialFunctionDeclaration{ Kind: common.DeclarationKindDestructorLegacy, FunctionDeclaration: &ast.FunctionDeclaration{ - Access: ast.AccessNotSpecified, - Comments: ast.EmptyComments, + Access: ast.AccessNotSpecified, Identifier: ast.Identifier{ Identifier: "destroy", Pos: ast.Position{Offset: 178, Line: 11, Column: 14}, diff --git a/old_parser/expression_test.go b/old_parser/expression_test.go index 5700783e2a..a7a533fdaf 100644 --- a/old_parser/expression_test.go +++ b/old_parser/expression_test.go @@ -33,6 +33,7 @@ import ( "github.com/onflow/cadence/ast" "github.com/onflow/cadence/common" "github.com/onflow/cadence/errors" + "github.com/onflow/cadence/old_parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) @@ -2133,6 +2134,71 @@ func TestParseBlockComment(t *testing.T) { ) }) + t.Run("invalid content", func(t *testing.T) { + + t.Parallel() + + // The lexer should never produce such an invalid token stream in the first place + + tokens := &testTokenStream{ + tokens: []lexer.Token{ + { + Type: lexer.TokenBlockCommentStart, + Range: ast.Range{ + StartPos: ast.Position{ + Line: 1, + Offset: 0, + Column: 0, + }, + EndPos: ast.Position{ + Line: 1, + Offset: 1, + Column: 1, + }, + }, + }, + { + Type: lexer.TokenIdentifier, + Range: ast.Range{ + StartPos: ast.Position{ + Line: 1, + Offset: 2, + Column: 2, + }, + EndPos: ast.Position{ + Line: 1, + Offset: 4, + Column: 4, + }, + }, + }, + {Type: lexer.TokenEOF}, + }, + input: []byte(`/*foo`), + } + + _, errs := ParseTokenStream( + nil, + tokens, + func(p *parser) (ast.Expression, error) { + return parseExpression(p, lowestBindingPower) + }, + Config{}, + ) + AssertEqualWithDiff(t, + []error{ + &SyntaxError{ + Message: "unexpected token identifier in block comment", + Pos: ast.Position{ + Line: 1, + Offset: 2, + Column: 2, + }, + }, + }, + errs, + ) + }) } func BenchmarkParseInfix(b *testing.B) { diff --git a/old_parser/function.go b/old_parser/function.go index fc7256711f..f6db3a30a9 100644 --- a/old_parser/function.go +++ b/old_parser/function.go @@ -297,8 +297,8 @@ func parseFunctionDeclaration( nativePos *ast.Position, docString string, ) (*ast.FunctionDeclaration, error) { - startToken := p.current - startPos := ast.EarliestPosition(startToken.StartPos, accessPos, staticPos, nativePos) + + startPos := ast.EarliestPosition(p.current.StartPos, accessPos, staticPos, nativePos) // Skip the `fun` keyword p.nextSemanticToken() diff --git a/old_parser/parser.go b/old_parser/parser.go index 59e5ba0fb0..975c44ecfd 100644 --- a/old_parser/parser.go +++ b/old_parser/parser.go @@ -19,6 +19,7 @@ package old_parser import ( + "bytes" "os" "strings" @@ -420,6 +421,9 @@ func (p *parser) skipSpaceAndComments() (containsNewline bool) { return } +var blockCommentDocStringPrefix = []byte("/**") +var lineCommentDocStringPrefix = []byte("///") + func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docString string) { var docStringBuilder strings.Builder defer func() { @@ -428,7 +432,7 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt } }() - var atEnd bool + var atEnd, insideLineDocString bool for !atEnd { switch p.current.Type { @@ -449,6 +453,43 @@ func (p *parser) parseTrivia(options triviaOptions) (containsNewline bool, docSt p.next() + case lexer.TokenBlockCommentStart: + commentStartOffset := p.current.StartPos.Offset + endToken, ok := p.parseBlockComment() + + if ok && options.parseDocStrings { + commentEndOffset := endToken.EndPos.Offset + + contentWithPrefix := p.tokens.Input()[commentStartOffset : commentEndOffset-1] + + insideLineDocString = false + docStringBuilder.Reset() + if bytes.HasPrefix(contentWithPrefix, blockCommentDocStringPrefix) { + // Strip prefix (`/**`) + docStringBuilder.Write(contentWithPrefix[len(blockCommentDocStringPrefix):]) + } + } + + case lexer.TokenLineComment: + if options.parseDocStrings { + comment := p.currentTokenSource() + if bytes.HasPrefix(comment, lineCommentDocStringPrefix) { + if insideLineDocString { + docStringBuilder.WriteByte('\n') + } else { + insideLineDocString = true + docStringBuilder.Reset() + } + // Strip prefix + docStringBuilder.Write(comment[len(lineCommentDocStringPrefix):]) + } else { + insideLineDocString = false + docStringBuilder.Reset() + } + } + + p.next() + default: atEnd = true } diff --git a/old_parser/statement.go b/old_parser/statement.go index 7f2ea66cd9..ae8f6d5980 100644 --- a/old_parser/statement.go +++ b/old_parser/statement.go @@ -155,7 +155,7 @@ func parseStatement(p *parser) (ast.Statement, error) { func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.Statement, error) { - startToken := p.current + startPos := p.current.StartPos // Skip the `fun` keyword p.nextSemanticToken() @@ -193,7 +193,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State parameterList, returnTypeAnnotation, functionBlock, - startToken.StartPos, + startPos, ast.EmptyComments, ), nil } else { @@ -211,7 +211,7 @@ func parseFunctionDeclarationOrFunctionExpressionStatement(p *parser) (ast.State parameterList, returnTypeAnnotation, functionBlock, - startToken.StartPos, + startPos, ), ), nil } From 99de32620aa2d50e79c3e0529824c87074c8585c Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Fri, 21 Nov 2025 19:46:20 +0100 Subject: [PATCH 64/84] remove NewNominalTypeWithComments --- ast/access_test.go | 6 +++--- ast/attachment_test.go | 2 ++ ast/type.go | 13 ------------- bbq/compiler/desugar.go | 2 ++ old_parser/declaration_test.go | 3 +++ old_parser/type.go | 1 + parser/declaration_test.go | 1 + parser/type.go | 3 ++- sema/checker.go | 2 ++ ...adence_v0.42_to_v1_contract_upgrade_validator.go | 7 ++++--- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/ast/access_test.go b/ast/access_test.go index cd706b04f3..0ad41031ae 100644 --- a/ast/access_test.go +++ b/ast/access_test.go @@ -43,7 +43,7 @@ func TestMappedAccess_MarshalJSON(t *testing.T) { t.Parallel() - e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}) + e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}, EmptyComments) access := NewMappedAccess(e, Position{Offset: 0, Line: 0, Column: 0}) actual, err := json.Marshal(access) @@ -68,8 +68,8 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { t.Parallel() - e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 0, Line: 0, Column: 0}), []Identifier{}) - f := NewNominalType(nil, NewIdentifier(nil, "F", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}) + e := NewNominalType(nil, NewIdentifier(nil, "E", Position{Offset: 0, Line: 0, Column: 0}), []Identifier{}, EmptyComments) + f := NewNominalType(nil, NewIdentifier(nil, "F", Position{Offset: 1, Line: 2, Column: 3}), []Identifier{}, EmptyComments) t.Run("conjunction", func(t *testing.T) { t.Parallel() diff --git a/ast/attachment_test.go b/ast/attachment_test.go index 9442f2532a..ffd7271f6a 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -46,6 +46,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { Position{Offset: 1, Line: 2, Column: 3}, ), []Identifier{}, + EmptyComments, ), Conformances: []*NominalType{ { @@ -450,6 +451,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { Position{Offset: 1, Line: 2, Column: 3}, ), []Identifier{}, + EmptyComments, ), Value: NewIdentifierExpression( nil, diff --git a/ast/type.go b/ast/type.go index d28863ea61..6d3c4bb750 100644 --- a/ast/type.go +++ b/ast/type.go @@ -120,19 +120,6 @@ func NewNominalType( memoryGauge common.MemoryGauge, identifier Identifier, nestedIdentifiers []Identifier, -) *NominalType { - common.UseMemory(memoryGauge, common.NominalTypeMemoryUsage) - return &NominalType{ - Identifier: identifier, - NestedIdentifiers: nestedIdentifiers, - } -} - -// TODO(preserve-comments): Should we remove this and use only NewNominalType (requires updating all dependants) -func NewNominalTypeWithComments( - memoryGauge common.MemoryGauge, - identifier Identifier, - nestedIdentifiers []Identifier, comments Comments, ) *NominalType { common.UseMemory(memoryGauge, common.NominalTypeMemoryUsage) diff --git a/bbq/compiler/desugar.go b/bbq/compiler/desugar.go index d9959d1cf4..1246bede47 100644 --- a/bbq/compiler/desugar.go +++ b/bbq/compiler/desugar.go @@ -1849,6 +1849,7 @@ func newEnumInitializer( ast.EmptyPosition, ), nil, + ast.EmptyComments, ), ast.EmptyPosition, ), @@ -2003,6 +2004,7 @@ func newEnumLookup( ast.EmptyPosition, ), nil, + ast.EmptyComments, ), ast.EmptyPosition, ), diff --git a/old_parser/declaration_test.go b/old_parser/declaration_test.go index 06eea2fbc5..c7466c1957 100644 --- a/old_parser/declaration_test.go +++ b/old_parser/declaration_test.go @@ -2959,6 +2959,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 24, Offset: 24}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ @@ -3001,6 +3002,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 24, Offset: 24}, }, nil, + ast.EmptyComments, ), ast.NewNominalType( nil, @@ -3009,6 +3011,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 28, Offset: 28}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ diff --git a/old_parser/type.go b/old_parser/type.go index d04f8cbd88..e7f97c6949 100644 --- a/old_parser/type.go +++ b/old_parser/type.go @@ -219,6 +219,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, + ast.EmptyComments, ), nil } diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 9b350d7664..9e8c336f35 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -6124,6 +6124,7 @@ func TestParseAttachmentDeclaration(t *testing.T) { Pos: ast.Position{Line: 1, Column: 32, Offset: 32}, }, nil, + ast.EmptyComments, ), }, Range: ast.Range{ diff --git a/parser/type.go b/parser/type.go index 4770a763c9..425597ef09 100644 --- a/parser/type.go +++ b/parser/type.go @@ -208,7 +208,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, trailingComments = nestedToken.Comments.PackToList() } - return ast.NewNominalTypeWithComments( + return ast.NewNominalType( p.memoryGauge, p.tokenToIdentifier(token), nestedIdentifiers, @@ -1100,6 +1100,7 @@ func parseFunctionType(p *parser, startPos ast.Position, purity ast.FunctionPuri endPos, ), nil, + ast.EmptyComments, ) returnTypeAnnotation = ast.NewTypeAnnotation( p.memoryGauge, diff --git a/sema/checker.go b/sema/checker.go index 924c52b63d..676dfa266c 100644 --- a/sema/checker.go +++ b/sema/checker.go @@ -1244,6 +1244,7 @@ func (checker *Checker) convertNominalType(t *ast.NominalType) Type { checker.memoryGauge, t.Identifier, resolvedIdentifiers, + ast.EmptyComments, ), }, ) @@ -1259,6 +1260,7 @@ func (checker *Checker) convertNominalType(t *ast.NominalType) Type { checker.memoryGauge, t.Identifier, resolvedIdentifiers, + ast.EmptyComments, ) checker.report( &NotDeclaredError{ diff --git a/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go b/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go index ab854202b0..2d8afb09d6 100644 --- a/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go +++ b/stdlib/cadence_v0.42_to_v1_contract_upgrade_validator.go @@ -309,7 +309,7 @@ func (validator *CadenceV042ToV1ContractUpdateValidator) expectedAuthorizationOf } func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRepresentableComposite(newDecl *ast.CompositeDeclaration) { - dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil) + dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil, ast.EmptyComments) compositeType := validator.getCompositeType(dummyNominalType) supportedEntitlements := compositeType.SupportedEntitlements() @@ -323,7 +323,7 @@ func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRep } func (validator *CadenceV042ToV1ContractUpdateValidator) validateEntitlementsRepresentableInterface(newDecl *ast.InterfaceDeclaration) { - dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil) + dummyNominalType := ast.NewNominalType(nil, newDecl.Identifier, nil, ast.EmptyComments) interfaceType := validator.getInterfaceType(dummyNominalType) supportedEntitlements := interfaceType.SupportedEntitlements() @@ -865,7 +865,7 @@ func semaConformanceToASTNominalType(newConformance sema.Conformance) *ast.Nomin } if containerType == nil { - return ast.NewNominalType(nil, identifier, nil) + return ast.NewNominalType(nil, identifier, nil, ast.EmptyComments) } return ast.NewNominalType( @@ -874,6 +874,7 @@ func semaConformanceToASTNominalType(newConformance sema.Conformance) *ast.Nomin Identifier: containerType.String(), }, []ast.Identifier{identifier}, + ast.EmptyComments, ) } From f0eb847808caedf4f9c02ab58583b6009b18f365 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Fri, 21 Nov 2025 20:06:58 +0100 Subject: [PATCH 65/84] resolve small todos --- ast/comments.go | 6 +++--- ast/function_declaration_test.go | 1 - parser/comment_test.go | 3 +-- parser/lexer/state.go | 1 - parser/parser.go | 1 - parser/statement.go | 5 ++--- 6 files changed, 6 insertions(+), 11 deletions(-) diff --git a/ast/comments.go b/ast/comments.go index ddeae5f185..2d0d525a69 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -25,7 +25,7 @@ func (c Comments) PackToList() []*Comment { func (c Comments) LeadingDocString() string { var s strings.Builder for _, comment := range c.Leading { - if comment.Doc() { + if comment.IsDoc() { if s.Len() > 0 { s.WriteRune('\n') } @@ -40,7 +40,7 @@ type Comment struct { } func NewComment(memoryGauge common.MemoryGauge, source []byte) *Comment { - // TODO(preserve-comments): Track memory usage + common.UseMemory(memoryGauge, common.NewRawStringMemoryUsage(len(source))) return &Comment{ source: source, } @@ -56,7 +56,7 @@ func (c Comment) Multiline() bool { return bytes.HasPrefix(c.source, blockCommentStringPrefix) } -func (c Comment) Doc() bool { +func (c Comment) IsDoc() bool { if c.Multiline() { return bytes.HasPrefix(c.source, blockCommentDocStringPrefix) } else { diff --git a/ast/function_declaration_test.go b/ast/function_declaration_test.go index 3b71402244..87c7a067fe 100644 --- a/ast/function_declaration_test.go +++ b/ast/function_declaration_test.go @@ -597,7 +597,6 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { actual, err := json.Marshal(decl) require.NoError(t, err) - // TODO(preserve-comments): Do we need to include comments in the JSON AST? assert.JSONEq(t, // language=json ` diff --git a/parser/comment_test.go b/parser/comment_test.go index 90b2d96307..5b010328fc 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -77,8 +77,7 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() - // TODO(preserve-comments): Extracting comments from operator tokens is a bit difficult, - // so let's handle this later as it seems a pretty edge case location to add comments. + // Extracting comments attached to the infix operator is more difficult and also an edge case, so ignore that for now. result, errs := testParseExpression(" 1/*test foo*/+/* bar */ 2 ") require.Empty(t, errs) diff --git a/parser/lexer/state.go b/parser/lexer/state.go index c7c82e2443..59f4635a00 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -278,7 +278,6 @@ func spaceState(startIsNewline bool) stateFn { l.scanSpace() - // TODO(preserve-comments): Do we need to track memory for other token types as well? common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) if containsNewline { diff --git a/parser/parser.go b/parser/parser.go index 0c42d6c47c..d90efe1bac 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -503,7 +503,6 @@ func (p *parser) tokenToIdentifier(token lexer.Token) ast.Identifier { p.memoryGauge, string(p.tokenSource(token)), token.StartPos, - // TODO(preserve-comments): Handle comments for identifiers (attach them to parent structs?) ) } diff --git a/parser/statement.go b/parser/statement.go index e061a4e0cf..7419061606 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -541,8 +541,7 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { expression, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), - // TODO(preserve-comments): handle inToken=nil + Leading: startToken.Comments.PackToList(), Trailing: inToken.Comments.PackToList(), }, ), nil @@ -1012,8 +1011,8 @@ func parseRemoveStatement( } leadingComments := startToken.Comments.PackToList() - // TODO(preserve-comments): Handle fromToken=nil leadingComments = append(leadingComments, fromToken.Comments.PackToList()...) + return ast.NewRemoveStatement( p.memoryGauge, attachmentNominalType, From dcd9b8bce7c40d0aa062d8b25c1be0cbbc3d6212 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Fri, 21 Nov 2025 21:20:11 +0100 Subject: [PATCH 66/84] cleanup comments parsing in lexer --- ast/comments.go | 3 +- parser/declaration.go | 10 ++--- parser/function.go | 8 ++-- parser/lexer/lexer.go | 91 +++++++++++++++++++++++-------------------- parser/lexer/state.go | 3 +- parser/lexer/token.go | 10 ++--- parser/statement.go | 40 +++++++++---------- parser/transaction.go | 2 +- parser/type.go | 2 +- 9 files changed, 88 insertions(+), 81 deletions(-) diff --git a/ast/comments.go b/ast/comments.go index 2d0d525a69..adc0bc7238 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -14,7 +14,8 @@ type Comments struct { var EmptyComments = Comments{} -func (c Comments) PackToList() []*Comment { +// All combines Leading and Trailing comments in a single array. +func (c Comments) All() []*Comment { var comments []*Comment comments = append(comments, c.Leading...) comments = append(comments, c.Trailing...) diff --git a/parser/declaration.go b/parser/declaration.go index d6cefedb8f..d29d91dccc 100644 --- a/parser/declaration.go +++ b/parser/declaration.go @@ -514,9 +514,9 @@ func parseVariableDeclaration( } var leadingComments []*ast.Comment - leadingComments = append(leadingComments, startToken.Comments.PackToList()...) - leadingComments = append(leadingComments, identifierToken.PackToList()...) - leadingComments = append(leadingComments, transferToken.PackToList()...) + leadingComments = append(leadingComments, startToken.Comments.All()...) + leadingComments = append(leadingComments, identifierToken.Comments.All()...) + leadingComments = append(leadingComments, transferToken.Comments.All()...) variableDeclaration := ast.NewVariableDeclaration( p.memoryGauge, @@ -834,7 +834,7 @@ func parseImportDeclaration(p *parser) (*ast.ImportDeclaration, error) { ), locationPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), + Leading: startToken.Comments.All(), Trailing: trailingComments, }, ), nil @@ -2071,7 +2071,7 @@ func parseEnumCase( p.memoryGauge, access, identifier, - ast.Comments{Leading: startToken.Comments.PackToList()}, + ast.Comments{Leading: startToken.Comments.All()}, startToken.StartPos, ), nil } diff --git a/parser/function.go b/parser/function.go index 522c3ddad6..17b37ac3ea 100644 --- a/parser/function.go +++ b/parser/function.go @@ -115,7 +115,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL if len(parameters) == 0 { comments.Leading = append( comments.Leading, - startToken.Comments.PackToList()..., + startToken.Comments.All()..., ) } else { comments.Leading = append( @@ -130,7 +130,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } comments.Trailing = append( comments.Trailing, - endToken.Comments.PackToList()..., + endToken.Comments.All()..., ) return ast.NewParameterList( @@ -225,8 +225,8 @@ func parseParameter(p *parser, expectDefaultArgument bool) (*ast.Parameter, erro startToken.StartPos, ast.Comments{ Leading: append( - startToken.Comments.PackToList(), - colonToken.Comments.PackToList()..., + startToken.Comments.All(), + colonToken.Comments.All()..., ), }, ), nil diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index c0b4182618..9594234cb9 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -56,6 +56,9 @@ const ( lexerModeStringInterpolation ) +// trailingCommentsEndMarker is a sentinel value for determining the end of trailing comments +var trailingCommentsEndMarker *ast.Comment = nil + type lexer struct { // memoryGauge is used for metering memory usage memoryGauge common.MemoryGauge @@ -204,7 +207,7 @@ func (l *lexer) run(state stateFn) (err error) { state = state(l) } - l.updatePreviousTrailingComments() + l.setPrevTokenTrailingComments() return } @@ -273,18 +276,6 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos := l.endPos() - // Only track trivia for non-space tokens - var leadingComments []*ast.Comment - shouldConsumeTrivia := ty != TokenSpace - if shouldConsumeTrivia { - l.updatePreviousTrailingComments() - leadingComments = l.currentComments - defer (func() { - // Mark as fully consumed - l.currentComments = []*ast.Comment{} - })() - } - token := Token{ Type: ty, SpaceOrError: spaceOrError, @@ -298,11 +289,7 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co endPos.column, ), ), - Comments: ast.Comments{ - Leading: leadingComments, - // Trailing comments can't be determined, as it wasn't consumed yet at this point. - Trailing: nil, - }, + Comments: l.tokenComments(ty), } l.tokens = append(l.tokens, token) @@ -313,31 +300,56 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co } } -func (l *lexer) updatePreviousTrailingComments() { +// tokenComments creates initial ast.Comments with leading comments, but empty trailing comments. +// Trailing comments for the current token are determined when the next non-space token is consumed (in setPrevTokenTrailingComments). +func (l *lexer) tokenComments(ty TokenType) ast.Comments { + if ty == TokenSpace { + return ast.EmptyComments + } + + l.setPrevTokenTrailingComments() + + leadingComments := l.currentComments + defer (func() { + // Mark as fully consumed + l.currentComments = []*ast.Comment{} + })() + + return ast.Comments{ + Leading: leadingComments, + // Trailing comments can't be determined yet, since we haven't consumed them. + Trailing: nil, + } +} + +// setPrevTokenTrailingComments sets the trailing comments of the latest non-space token, +// since they were not known yet at the time of calling tokenComments. +func (l *lexer) setPrevTokenTrailingComments() { if l.tokenCount == 0 { return } - lastNonSpaceTokenIndex := -1 + latestNonSpaceTokenIndex := -1 for i := l.tokenCount - 1; i >= 0; i-- { if l.tokens[i].Type != TokenSpace { - lastNonSpaceTokenIndex = i + latestNonSpaceTokenIndex = i break } } - // Split the current comment into trailing comment of the previous token - // and leading comment of the next token. - var trailing []*ast.Comment - var leading []*ast.Comment - - trailingTriviaEnded := lastNonSpaceTokenIndex == -1 + // Split the current comments into two sets: + // - trailing comments of the previous token + // - leading comments of the next token + var trailing, leading []*ast.Comment + // If all previously seen tokens are spaces, + // we treat all comments as the leading set of the current token. + isLeadingSet := latestNonSpaceTokenIndex == -1 for _, comment := range l.currentComments { - if comment == nil { - trailingTriviaEnded = true + if comment == trailingCommentsEndMarker { + isLeadingSet = true continue } - if trailingTriviaEnded { + if isLeadingSet { leading = append(leading, comment) } else { trailing = append(trailing, comment) @@ -346,23 +358,18 @@ func (l *lexer) updatePreviousTrailingComments() { l.currentComments = leading - if lastNonSpaceTokenIndex != -1 { - lastNonSpaceToken := &l.tokens[lastNonSpaceTokenIndex] - - if len(trailing) == 0 { - return - } - - if lastNonSpaceToken.Trailing == nil { - lastNonSpaceToken.Trailing = []*ast.Comment{} + if latestNonSpaceTokenIndex != -1 && len(trailing) > 0 { + lastNonSpaceToken := &l.tokens[latestNonSpaceTokenIndex] + if lastNonSpaceToken.Comments.Trailing == nil { + lastNonSpaceToken.Comments.Trailing = []*ast.Comment{} } - lastNonSpaceToken.Trailing = append(lastNonSpaceToken.Trailing, trailing...) + lastNonSpaceToken.Comments.Trailing = append(lastNonSpaceToken.Comments.Trailing, trailing...) } } -func (l *lexer) emitNewlineSentinelComment() { - l.currentComments = append(l.currentComments, nil) +func (l *lexer) markTrailingCommentsEnd() { + l.currentComments = append(l.currentComments, trailingCommentsEndMarker) } func (l *lexer) emitComment() { diff --git a/parser/lexer/state.go b/parser/lexer/state.go index 59f4635a00..5f014eef31 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -281,7 +281,8 @@ func spaceState(startIsNewline bool) stateFn { common.UseMemory(l.memoryGauge, common.SpaceTokenMemoryUsage) if containsNewline { - l.emitNewlineSentinelComment() + // Trailing comments end before the first newline. + l.markTrailingCommentsEnd() } l.emit( diff --git a/parser/lexer/token.go b/parser/lexer/token.go index 8bf7b47289..4f8e842138 100644 --- a/parser/lexer/token.go +++ b/parser/lexer/token.go @@ -26,12 +26,10 @@ type Token struct { SpaceOrError any Type TokenType ast.Range - // TODO(preserve-comments): This is currently not true (first comment), - // as leading comments are just all comments after the trailing comments of the previous token. - // Leading comments span up to and including the first contiguous sequence of newlines characters. - // Trailing comments span up to, but not including, the next newline character. - // Not tracked for space token, since those are usually ignored in the parser. - ast.Comments + // Leading comments span from the end of the trailing comments of the last (non-space) token to the start of the next (non-space) token. + // Trailing comments span up to, but not including, the next newline character after the current (non-space) token. + // Important: comments are not tracked for space token, since those are usually ignored in the parser. + Comments ast.Comments } func (t Token) Is(ty TokenType) bool { diff --git a/parser/statement.go b/parser/statement.go index 7419061606..88a6b82ea5 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -193,10 +193,10 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( startPos = funToken.StartPos } else { startPos = *ast.EarlierPosition(&funToken.StartPos, &purityToken.StartPos) - leadingComments = append(leadingComments, purityToken.Comments.PackToList()...) + leadingComments = append(leadingComments, purityToken.Comments.All()...) } - leadingComments = append(leadingComments, funToken.Comments.PackToList()...) + leadingComments = append(leadingComments, funToken.Comments.All()...) // Skip the `fun` keyword p.nextSemanticToken() @@ -295,8 +295,8 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { if endToken == nil { comments = startToken.Comments } else { - comments.Leading = startToken.Comments.PackToList() - comments.Trailing = endToken.Comments.PackToList() + comments.Leading = startToken.Comments.All() + comments.Trailing = endToken.Comments.All() } return ast.NewReturnStatement( @@ -387,7 +387,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { // The parser ignores the `else` token, // so to preserve potential comments associated with else token, // we attach the comments to the (next) `if` token. - leadingComments := elseToken.Comments.PackToList() + leadingComments := elseToken.Comments.All() leadingComments = append(leadingComments, p.current.Comments.Leading...) p.current.Comments.Leading = leadingComments @@ -418,7 +418,7 @@ func parseIfStatement(p *parser) (*ast.IfStatement, error) { elseBlock, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), + Leading: startToken.Comments.All(), }, ) @@ -472,7 +472,7 @@ func parseWhileStatement(p *parser) (*ast.WhileStatement, error) { block, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), + Leading: startToken.Comments.All(), }, ), nil } @@ -541,8 +541,8 @@ func parseForStatement(p *parser) (*ast.ForStatement, error) { expression, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), - Trailing: inToken.Comments.PackToList(), + Leading: startToken.Comments.All(), + Trailing: inToken.Comments.All(), }, ), nil } @@ -612,8 +612,8 @@ func parseBlock(p *parser) (*ast.Block, error) { endToken.EndPos, ), ast.Comments{ - Leading: startToken.Comments.PackToList(), - Trailing: endToken.Comments.PackToList(), + Leading: startToken.Comments.All(), + Trailing: endToken.Comments.All(), }, ), nil } @@ -682,8 +682,8 @@ func parseFunctionBlock(p *parser) (*ast.FunctionBlock, error) { endToken.EndPos, ), ast.Comments{ - Leading: startToken.Leading, - Trailing: endToken.Trailing, + Leading: startToken.Comments.Leading, + Trailing: endToken.Comments.Trailing, }, ), res.preConditions, @@ -798,7 +798,7 @@ func parseEmitStatement(p *parser) (*ast.EmitStatement, error) { invocation, startToken.StartPos, ast.Comments{ - Leading: startToken.Comments.PackToList(), + Leading: startToken.Comments.All(), }, ), nil } @@ -836,8 +836,8 @@ func parseSwitchStatement(p *parser) (*ast.SwitchStatement, error) { endToken.EndPos, ), ast.Comments{ - Leading: startToken.Comments.PackToList(), - Trailing: endToken.Comments.PackToList(), + Leading: startToken.Comments.All(), + Trailing: endToken.Comments.All(), }, ), nil } @@ -966,8 +966,8 @@ func parseSwitchCase(p *parser, hasExpression bool) (*ast.SwitchCase, error) { endPos, ), ast.Comments{ - Leading: startToken.Comments.PackToList(), - Trailing: colonToken.Comments.PackToList(), + Leading: startToken.Comments.All(), + Trailing: colonToken.Comments.All(), }, ), nil } @@ -1010,8 +1010,8 @@ func parseRemoveStatement( return nil, err } - leadingComments := startToken.Comments.PackToList() - leadingComments = append(leadingComments, fromToken.Comments.PackToList()...) + leadingComments := startToken.Comments.All() + leadingComments = append(leadingComments, fromToken.Comments.All()...) return ast.NewRemoveStatement( p.memoryGauge, diff --git a/parser/transaction.go b/parser/transaction.go index 92293ef1f9..878f9d2c43 100644 --- a/parser/transaction.go +++ b/parser/transaction.go @@ -301,7 +301,7 @@ func parseTransactionExecute(p *parser) (*ast.SpecialFunctionDeclaration, error) ), identifier.Pos, ast.Comments{ - Leading: identifierToken.Leading, + Leading: identifierToken.Comments.Leading, }, ), ), nil diff --git a/parser/type.go b/parser/type.go index 425597ef09..81b409ab1b 100644 --- a/parser/type.go +++ b/parser/type.go @@ -205,7 +205,7 @@ func parseNominalTypeRemainder(p *parser, token lexer.Token) (*ast.NominalType, if nestedToken == nil { trailingComments = token.Comments.Trailing } else { - trailingComments = nestedToken.Comments.PackToList() + trailingComments = nestedToken.Comments.All() } return ast.NewNominalType( From 64663e3285c977f9478ccff738a128351844ba94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 21 Nov 2025 14:18:44 -0800 Subject: [PATCH 67/84] fix missing comment end error --- cmd/errors/errors.go | 5 ----- parser/lexer/lexer_test.go | 2 +- parser/lexer/state.go | 10 +++++++++- parser/parser.go | 17 +++++++++++++++-- 4 files changed, 25 insertions(+), 9 deletions(-) diff --git a/cmd/errors/errors.go b/cmd/errors/errors.go index b1d2ce0a56..473ad47b11 100644 --- a/cmd/errors/errors.go +++ b/cmd/errors/errors.go @@ -589,11 +589,6 @@ func generateErrors() []namedError { Token: placeholderToken, }, }, - {"parser.UnexpectedTokenInBlockCommentError", - &parser.UnexpectedTokenInBlockCommentError{ - GotToken: placeholderToken, - }, - }, {"parser.UnexpectedTokenInExpressionError", &parser.UnexpectedTokenInExpressionError{ GotToken: placeholderToken, diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 0a9b49aab7..55556cad05 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -2658,7 +2658,7 @@ func TestLexBlockComment(t *testing.T) { { Token: Token{ Type: TokenError, - SpaceOrError: errors.New(`missing comment end '*/'`), + SpaceOrError: MissingCommentEndError{}, Range: ast.Range{ StartPos: ast.Position{Line: 1, Column: 20, Offset: 20}, EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, diff --git a/parser/lexer/state.go b/parser/lexer/state.go index c7c82e2443..81462aaf35 100644 --- a/parser/lexer/state.go +++ b/parser/lexer/state.go @@ -338,7 +338,7 @@ func blockCommentState(l *lexer, nesting int) stateFn { r := l.next() switch r { case EOF: - l.emitError(fmt.Errorf("missing comment end '*/'")) + l.emitError(MissingCommentEndError{}) return nil case '/': beforeSlashOffset := l.prevEndOffset @@ -368,3 +368,11 @@ func blockCommentState(l *lexer, nesting int) stateFn { return blockCommentState(l, nesting) } } + +type MissingCommentEndError struct{} + +var _ error = MissingCommentEndError{} + +func (MissingCommentEndError) Error() string { + return "missing comment end (`*/`)" +} diff --git a/parser/parser.go b/parser/parser.go index 0c42d6c47c..917b7014ee 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -254,13 +254,26 @@ func (p *parser) next() { if !ok { panic(errors.NewUnreachableError()) } - parseError, ok := err.(ParseError) - if !ok { + + var parseError ParseError + switch err := err.(type) { + case ParseError: + // already a parse error, do nothing + parseError = err + + case lexer.MissingCommentEndError: + parseError = &MissingCommentEndError{ + Pos: token.StartPos, + } + + default: + // wrap other errors as syntax errors parseError = &SyntaxError{ Pos: token.StartPos, Message: err.Error(), } } + p.report(parseError) continue } From 96eb1d265ecb407f492e986d1d399ea20c4de49e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bastian=20M=C3=BCller?= Date: Fri, 21 Nov 2025 14:19:15 -0800 Subject: [PATCH 68/84] remove UnexpectedTokenInBlockCommentError --- parser/comment_test.go | 47 ------------------------------------------ parser/errors.go | 38 ---------------------------------- 2 files changed, 85 deletions(-) diff --git a/parser/comment_test.go b/parser/comment_test.go index 90b2d96307..6eeb854549 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -27,7 +27,6 @@ import ( "github.com/onflow/cadence/errors" "github.com/onflow/cadence/ast" - "github.com/onflow/cadence/parser/lexer" . "github.com/onflow/cadence/test_utils/common_utils" ) @@ -194,52 +193,6 @@ func TestParseBlockComment(t *testing.T) { errs, ) }) - - t.Run("invalid content", func(t *testing.T) { - - t.Parallel() - - // The lexer should never produce such an invalid token stream in the first place - - tokens := &testTokenStream{ - tokens: []lexer.Token{ - // TODO(merge): is this correct? - { - Type: lexer.TokenIdentifier, - Range: ast.Range{ - StartPos: ast.Position{Line: 1, Offset: 2, Column: 2}, - EndPos: ast.Position{Line: 1, Offset: 4, Column: 4}, - }, - }, - {Type: lexer.TokenEOF}, - }, - input: []byte(`/*foo`), - } - - // TODO(merge): move and emit UnexpectedTokenInBlockCommentError from lexer - _, errs := ParseTokenStream( - nil, - tokens, - func(p *parser) (ast.Expression, error) { - return parseExpression(p, lowestBindingPower) - }, - Config{}, - ) - AssertEqualWithDiff(t, - []error{ - &UnexpectedTokenInBlockCommentError{ - GotToken: lexer.Token{ - Range: ast.Range{ - StartPos: ast.Position{Offset: 2, Line: 1, Column: 2}, - EndPos: ast.Position{Offset: 4, Line: 1, Column: 4}, - }, - Type: lexer.TokenIdentifier, - }, - }, - }, - errs, - ) - }) } func TestParseWhileStatementComment(t *testing.T) { diff --git a/parser/errors.go b/parser/errors.go index 19c66a622c..8cba3d8749 100644 --- a/parser/errors.go +++ b/parser/errors.go @@ -3975,7 +3975,6 @@ func (e *MissingCommentEndError) EndPosition(_ common.MemoryGauge) ast.Position return e.Pos } -// TODO(merge): Move and emit from lexer func (e *MissingCommentEndError) Error() string { return "missing comment end (`*/`)" } @@ -4005,43 +4004,6 @@ func (e *MissingCommentEndError) SuggestFixes(_ string) []errors.SuggestedFix[as } } -// UnexpectedTokenInBlockCommentError is reported when an unexpected token is found in a block comment. -type UnexpectedTokenInBlockCommentError struct { - GotToken lexer.Token -} - -var _ ParseError = &UnexpectedTokenInBlockCommentError{} -var _ errors.UserError = &UnexpectedTokenInBlockCommentError{} -var _ errors.SecondaryError = &UnexpectedTokenInBlockCommentError{} -var _ errors.HasDocumentationLink = &UnexpectedTokenInBlockCommentError{} - -func (*UnexpectedTokenInBlockCommentError) isParseError() {} - -func (*UnexpectedTokenInBlockCommentError) IsUserError() {} - -func (e *UnexpectedTokenInBlockCommentError) StartPosition() ast.Position { - return e.GotToken.StartPos -} - -func (e *UnexpectedTokenInBlockCommentError) EndPosition(_ common.MemoryGauge) ast.Position { - return e.GotToken.EndPos -} - -func (e *UnexpectedTokenInBlockCommentError) Error() string { - return fmt.Sprintf( - "unexpected token %s in block comment", - e.GotToken.Type, - ) -} - -func (*UnexpectedTokenInBlockCommentError) SecondaryError() string { - return "only text is allowed in a block comment" -} - -func (*UnexpectedTokenInBlockCommentError) DocumentationLink() string { - return "https://cadence-lang.org/docs/language/syntax#comments" -} - // SpecialFunctionReturnTypeError is reported when a special function has a return type. type SpecialFunctionReturnTypeError struct { DeclarationKind common.DeclarationKind From 3e177c568508bbf990a6188eab160b6738e9cbb9 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Sun, 23 Nov 2025 18:45:08 +0100 Subject: [PATCH 69/84] make comment tracking configurable --- parser/lexer/lexer.go | 33 +++++++++++++++++++++++++-------- parser/lexer/lexer_test.go | 2 +- parser/parser.go | 12 +++++++++++- parser/parser_test.go | 17 +++++++++++------ 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index 9594234cb9..af6de34ba1 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -88,8 +88,12 @@ type lexer struct { mode lexerMode // counts the number of unclosed brackets for string templates \((())) openBrackets int - // currentComments stores the current leading and/or trailing comments + // indicates if tracking comments is enabled + trackComments bool + // currentComments stores the leading and/or trailing comments, + // waiting to be consumed and attached to respective tokens // nil is used as sentinel value to track newlines + // Note: currentComments is only initialized when trackComments=true currentComments []*ast.Comment } @@ -176,6 +180,17 @@ func Lex(input []byte, memoryGauge common.MemoryGauge) (TokenStream, error) { return l, err } +// LexWithComments is equivalent to Lex, but it enables comments tracking. +func LexWithComments(input []byte, memoryGauge common.MemoryGauge) (TokenStream, error) { + l := pool.Get().(*lexer) + l.clear() + l.memoryGauge = memoryGauge + l.input = input + l.trackComments = true + err := l.run(rootState) + return l, err +} + // run executes the stateFn, which will scan the runes in the input // and emit tokens. // @@ -303,7 +318,7 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co // tokenComments creates initial ast.Comments with leading comments, but empty trailing comments. // Trailing comments for the current token are determined when the next non-space token is consumed (in setPrevTokenTrailingComments). func (l *lexer) tokenComments(ty TokenType) ast.Comments { - if ty == TokenSpace { + if ty == TokenSpace || !l.trackComments { return ast.EmptyComments } @@ -325,7 +340,7 @@ func (l *lexer) tokenComments(ty TokenType) ast.Comments { // setPrevTokenTrailingComments sets the trailing comments of the latest non-space token, // since they were not known yet at the time of calling tokenComments. func (l *lexer) setPrevTokenTrailingComments() { - if l.tokenCount == 0 { + if l.tokenCount == 0 || !l.trackComments { return } @@ -386,13 +401,15 @@ func (l *lexer) emitComment() { ), ) - if l.currentComments == nil { - l.currentComments = []*ast.Comment{} - } + l.consume(endPos) - l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, currentRange.Source(l.input))) + if l.trackComments { + if l.currentComments == nil { + l.currentComments = []*ast.Comment{} + } - l.consume(endPos) + l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, currentRange.Source(l.input))) + } } // endPos pre-computed end-position by calling l.endPos() diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 0a9b49aab7..b3273651eb 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -63,7 +63,7 @@ func testLex(t *testing.T, input string, expected []token) { bytes := []byte(input) - tokenStream, err := Lex(bytes, nil) + tokenStream, err := LexWithComments(bytes, nil) require.NoError(t, err) withTokens(tokenStream, func(actualTokens []Token) { diff --git a/parser/parser.go b/parser/parser.go index d90efe1bac..444cddf280 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -55,6 +55,8 @@ type Config struct { IgnoreLeadingIdentifierEnabled bool // TypeParametersEnabled determines if type parameters are enabled TypeParametersEnabled bool + // TrackComments enables comment tracking in the lexer and AST. + TrackComments bool } type parser struct { @@ -97,8 +99,16 @@ func Parse[T any]( parse func(*parser) (T, error), config Config, ) (result T, errors []error) { + lex := func() (lexer.TokenStream, error) { + if config.TrackComments { + return lexer.LexWithComments(input, memoryGauge) + } else { + return lexer.Lex(input, memoryGauge) + } + } + // create a lexer, which turns the input string into tokens - tokens, err := lexer.Lex(input, memoryGauge) + tokens, err := lex() if err != nil { errors = append(errors, err) return diff --git a/parser/parser_test.go b/parser/parser_test.go index 2e3be4fb78..92a77d72c1 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -105,7 +105,7 @@ func testParseStatements(s string) ([]ast.Statement, []error) { } func testParseStatementsWithConfig(s string, config Config) ([]ast.Statement, []error) { - statements, errs := ParseStatements(nil, []byte(s), config) + statements, errs := ParseStatements(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return statements, errs } @@ -115,7 +115,7 @@ func testParseDeclarations(s string) ([]ast.Declaration, []error) { } func testParseDeclarationsWithConfig(s string, config Config) ([]ast.Declaration, []error) { - declarations, errs := ParseDeclarations(nil, []byte(s), config) + declarations, errs := ParseDeclarations(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return declarations, errs } @@ -125,7 +125,7 @@ func testParseProgram(s string) (*ast.Program, error) { } func testParseProgramWithConfig(s string, config Config) (*ast.Program, error) { - program, err := ParseProgram(nil, []byte(s), config) + program, err := ParseProgram(nil, []byte(s), configWithCommentTracking(config)) if err != nil { _ = err.Error() } @@ -137,7 +137,7 @@ func testParseExpression(s string) (ast.Expression, []error) { } func testParseExpressionWithConfig(s string, config Config) (ast.Expression, []error) { - expression, errs := ParseExpression(nil, []byte(s), config) + expression, errs := ParseExpression(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return expression, errs } @@ -147,7 +147,7 @@ func testParseArgumentList(s string) (ast.Arguments, []error) { } func testParseArgumentListWithConfig(s string, config Config) (ast.Arguments, []error) { - arguments, errs := ParseArgumentList(nil, []byte(s), config) + arguments, errs := ParseArgumentList(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return arguments, errs } @@ -157,11 +157,16 @@ func testParseType(s string) (ast.Type, []error) { } func testParseTypeWithConfig(s string, config Config) (ast.Type, []error) { - ty, errs := ParseType(nil, []byte(s), config) + ty, errs := ParseType(nil, []byte(s), configWithCommentTracking(config)) checkErrorsPrintable(errs, s) return ty, errs } +func configWithCommentTracking(config Config) Config { + config.TrackComments = true + return config +} + func TestParseInvalid(t *testing.T) { t.Parallel() From 199294858461246a202656d1fd8e9986dff98658 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Sun, 23 Nov 2025 19:18:14 +0100 Subject: [PATCH 70/84] rename parser config field --- parser/parser.go | 6 +++--- parser/parser_test.go | 4 ++-- sema/gen/main.go | 7 ++++--- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 196a239cf6..7817d5062d 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -55,8 +55,8 @@ type Config struct { IgnoreLeadingIdentifierEnabled bool // TypeParametersEnabled determines if type parameters are enabled TypeParametersEnabled bool - // TrackComments enables comment tracking in the lexer and AST. - TrackComments bool + // CommentTrackingEnabled determines if comment should be parsed and tracked in lexer and parser. + CommentTrackingEnabled bool } type parser struct { @@ -100,7 +100,7 @@ func Parse[T any]( config Config, ) (result T, errors []error) { lex := func() (lexer.TokenStream, error) { - if config.TrackComments { + if config.CommentTrackingEnabled { return lexer.LexWithComments(input, memoryGauge) } else { return lexer.Lex(input, memoryGauge) diff --git a/parser/parser_test.go b/parser/parser_test.go index 92a77d72c1..a2a643c823 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -163,7 +163,7 @@ func testParseTypeWithConfig(s string, config Config) (ast.Type, []error) { } func configWithCommentTracking(config Config) Config { - config.TrackComments = true + config.CommentTrackingEnabled = true return config } @@ -1119,7 +1119,7 @@ func TestParseComments(t *testing.T) { event MyEvent() // After MyEvent /// Ignored `), - Config{}, + Config{CommentTrackingEnabled: true}, ) assert.Empty(t, errs) diff --git a/sema/gen/main.go b/sema/gen/main.go index ec3cbebc66..0bce2d1980 100644 --- a/sema/gen/main.go +++ b/sema/gen/main.go @@ -70,9 +70,10 @@ const headerTemplate = `// Code generated from {{ . }}. DO NOT EDIT. var parsedHeaderTemplate = template.Must(template.New("header").Parse(headerTemplate)) var parserConfig = parser.Config{ - StaticModifierEnabled: true, - NativeModifierEnabled: true, - TypeParametersEnabled: true, + StaticModifierEnabled: true, + NativeModifierEnabled: true, + TypeParametersEnabled: true, + CommentTrackingEnabled: true, } func initialUpper(s string) string { From a9352aff79cc445700ad4cbfd7b2e8fbe802b0e3 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Sun, 23 Nov 2025 19:46:14 +0100 Subject: [PATCH 71/84] fix lexer init in parser --- parser/parser.go | 20 ++++++++++---------- sema/gen/main_test.go | 7 +++++++ 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/parser/parser.go b/parser/parser.go index 7817d5062d..43445ed76e 100644 --- a/parser/parser.go +++ b/parser/parser.go @@ -99,16 +99,8 @@ func Parse[T any]( parse func(*parser) (T, error), config Config, ) (result T, errors []error) { - lex := func() (lexer.TokenStream, error) { - if config.CommentTrackingEnabled { - return lexer.LexWithComments(input, memoryGauge) - } else { - return lexer.Lex(input, memoryGauge) - } - } - // create a lexer, which turns the input string into tokens - tokens, err := lex() + tokens, err := lex(input, memoryGauge, config.CommentTrackingEnabled) if err != nil { errors = append(errors, err) return @@ -211,6 +203,14 @@ func ParseTokenStream[T any]( return result, p.errors } +func lex(input []byte, memoryGauge common.MemoryGauge, trackComments bool) (lexer.TokenStream, error) { + if trackComments { + return lexer.LexWithComments(input, memoryGauge) + } else { + return lexer.Lex(input, memoryGauge) + } +} + func (p *parser) newSyntaxError(message string, params ...any) *SyntaxError { return NewSyntaxError(p.current.StartPos, message, params...) } @@ -674,7 +674,7 @@ func ParseArgumentList( } func ParseProgram(memoryGauge common.MemoryGauge, code []byte, config Config) (program *ast.Program, err error) { - tokens, err := lexer.Lex(code, memoryGauge) + tokens, err := lex(code, memoryGauge, config.CommentTrackingEnabled) if err != nil { return } diff --git a/sema/gen/main_test.go b/sema/gen/main_test.go index d1ce9c277f..bce60f4353 100644 --- a/sema/gen/main_test.go +++ b/sema/gen/main_test.go @@ -51,6 +51,13 @@ func TestFiles(t *testing.T) { inputPath := filepath.Join(dirPath, "test.cdc") + defer func() { + err := recover() + if err != nil { + t.Errorf("%s should not have panicked: %s", testName, err) + } + }() + gen(inputPath, outFile, "github.com/onflow/cadence/sema/gen/"+dirPath) goldenPath := filepath.Join(dirPath, "test.golden.go") From 39571f274d63136474fadc87355cd150a0de4064 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 23 Nov 2025 20:08:59 +0100 Subject: [PATCH 72/84] avoid unecessary slice allocation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Müller --- ast/comments.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/ast/comments.go b/ast/comments.go index adc0bc7238..ffb0d494c1 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -65,17 +65,21 @@ func (c Comment) IsDoc() bool { } } +var commentPrefixes = [][]byte{ + blockCommentDocStringPrefix, // must be before blockCommentStringPrefix + blockCommentStringPrefix, + lineCommentDocStringPrefix, // must be before lineCommentStringPrefix + lineCommentStringPrefix, +} + +var commentSuffixes = [][]byte{ + blockCommentStringSuffix, +} + // Text without opening/closing comment characters /*, /**, */, // func (c Comment) Text() []byte { - withoutPrefixes := cutOptionalPrefixes(c.source, [][]byte{ - blockCommentDocStringPrefix, // must be before blockCommentStringPrefix - blockCommentStringPrefix, - lineCommentDocStringPrefix, // must be before lineCommentStringPrefix - lineCommentStringPrefix, - }) - return cutOptionalSuffixes(withoutPrefixes, [][]byte{ - blockCommentStringSuffix, - }) + withoutPrefixes := cutOptionalPrefixes(c.source, commentPrefixes) + return cutOptionalSuffixes(withoutPrefixes, commentSuffixes) } func cutOptionalPrefixes(input []byte, prefixes [][]byte) (output []byte) { From fb5324af94b2470e720e40a3bd594e523e7f30fa Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Sun, 23 Nov 2025 20:09:43 +0100 Subject: [PATCH 73/84] add back IsValidIdentifier --- parser/lexer/lexer.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index af6de34ba1..7e65b6a694 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -613,3 +613,15 @@ func (l *lexer) scanFixedPointRemainder() { func isDecimalDigitOrUnderscore(r rune) bool { return (r >= '0' && r <= '9') || r == '_' } + +func IsValidIdentifier(s string) bool { + // Note: func is used by downstream dependants, do not remove. + + for _, r := range s { + if !IsIdentifierRune(r) { + return false + } + } + + return true +} From e0f7aa4a7fe863d593ad119b345dd575e19ddfe3 Mon Sep 17 00:00:00 2001 From: Bart Date: Sun, 23 Nov 2025 20:28:19 +0100 Subject: [PATCH 74/84] break the arguments into multiple lines MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bastian Müller --- ast/block.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ast/block.go b/ast/block.go index f1fd53fe56..ad99a29226 100644 --- a/ast/block.go +++ b/ast/block.go @@ -34,7 +34,12 @@ type Block struct { var _ Element = &Block{} -func NewBlock(memoryGauge common.MemoryGauge, statements []Statement, astRange Range, comments Comments) *Block { +func NewBlock( + memoryGauge common.MemoryGauge, + statements []Statement, + astRange Range, + comments Comments, +) *Block { common.UseMemory(memoryGauge, common.BlockMemoryUsage) return &Block{ From 8f71c39be1c77cf8ea07c905d52c0db95025fe72 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 2 Dec 2025 12:58:31 +0100 Subject: [PATCH 75/84] add back empty comments test case, small allocation fix, add comment --- parser/comment_test.go | 7 +++++++ parser/lexer/lexer.go | 28 ++++++++++++++-------------- parser/lexer/lexer_test.go | 1 + 3 files changed, 22 insertions(+), 14 deletions(-) diff --git a/parser/comment_test.go b/parser/comment_test.go index 6e3943fb04..5f67418667 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -34,6 +34,13 @@ func TestParseBlockComment(t *testing.T) { t.Parallel() + t.Run("empty", func(t *testing.T) { + t.Parallel() + + _, errs := testParseExpression(`/**/ true`) + require.Empty(t, errs) + }) + t.Run("nested", func(t *testing.T) { t.Parallel() diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index 7e65b6a694..ea56fb6ced 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -390,26 +390,26 @@ func (l *lexer) markTrailingCommentsEnd() { func (l *lexer) emitComment() { endPos := l.endPos() - currentRange := ast.NewRange( - l.memoryGauge, - l.startPosition(), - ast.NewPosition( - l.memoryGauge, - l.endOffset-1, - endPos.line, - endPos.column, - ), - ) - - l.consume(endPos) - if l.trackComments { if l.currentComments == nil { l.currentComments = []*ast.Comment{} } - l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, currentRange.Source(l.input))) + commentRange := ast.NewRange( + l.memoryGauge, + l.startPosition(), + ast.NewPosition( + l.memoryGauge, + l.endOffset-1, + endPos.line, + endPos.column, + ), + ) + + l.currentComments = append(l.currentComments, ast.NewComment(l.memoryGauge, commentRange.Source(l.input))) } + + l.consume(endPos) } // endPos pre-computed end-position by calling l.endPos() diff --git a/parser/lexer/lexer_test.go b/parser/lexer/lexer_test.go index 5b973bfdbf..9bdd9c3b23 100644 --- a/parser/lexer/lexer_test.go +++ b/parser/lexer/lexer_test.go @@ -2664,6 +2664,7 @@ func TestLexBlockComment(t *testing.T) { EndPos: ast.Position{Line: 1, Column: 20, Offset: 20}, }, }, + // Error occurs at the EOF, which doesn't have a position within the source. Source: "\000", }, { From 094ec78f7d5d74b7e1bc6b29c8252895f52ed791 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 2 Dec 2025 12:59:30 +0100 Subject: [PATCH 76/84] reset trackComments in clear func --- parser/lexer/lexer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index ea56fb6ced..fe1d036864 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -157,6 +157,7 @@ func (l *lexer) clear() { l.mode = lexerModeNormal l.openBrackets = 0 l.currentComments = nil + l.trackComments = false } func (l *lexer) Reclaim() { From 73fd987a2968dd9f753a214fe52ecc889e8b9d05 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 2 Dec 2025 13:00:15 +0100 Subject: [PATCH 77/84] reuse existing comments slice allocation --- parser/lexer/lexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index fe1d036864..863500fe9e 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -156,7 +156,7 @@ func (l *lexer) clear() { l.tokenCount = 0 l.mode = lexerModeNormal l.openBrackets = 0 - l.currentComments = nil + l.currentComments = l.currentComments[:0] l.trackComments = false } From 6880089c9c3f3002cd4b1906e3aebb9a7aa58587 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 2 Dec 2025 13:03:54 +0100 Subject: [PATCH 78/84] remove unecessary if condition --- parser/statement.go | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/parser/statement.go b/parser/statement.go index 88a6b82ea5..4b5569a4b8 100644 --- a/parser/statement.go +++ b/parser/statement.go @@ -264,7 +264,6 @@ func parseFunctionDeclarationOrFunctionExpressionStatement( } func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { - var endToken *lexer.Token startToken := p.current tokenRange := startToken.Range endPosition := tokenRange.EndPos @@ -276,10 +275,14 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { var expression ast.Expression var err error + var endToken *lexer.Token + var comments ast.Comments switch p.current.Type { case lexer.TokenEOF, lexer.TokenSemicolon, lexer.TokenBraceClose: tok := p.current endToken = &tok + comments.Leading = startToken.Comments.All() + comments.Trailing = endToken.Comments.All() default: if !sawNewLine { expression, err = parseExpression(p, lowestBindingPower) @@ -289,14 +292,7 @@ func parseReturnStatement(p *parser) (*ast.ReturnStatement, error) { endPosition = expression.EndPosition(p.memoryGauge) } - } - - comments := ast.EmptyComments - if endToken == nil { comments = startToken.Comments - } else { - comments.Leading = startToken.Comments.All() - comments.Trailing = endToken.Comments.All() } return ast.NewReturnStatement( From a91c792b759c4cc5b08d498dbc18983464a2efe5 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Tue, 9 Dec 2025 22:57:49 +0100 Subject: [PATCH 79/84] fix tests --- parser/declaration_test.go | 4 +++- parser/lexer/lexer.go | 11 ++++++++--- parser/parser_test.go | 4 +++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 9e8c336f35..c83a87730e 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -548,7 +548,9 @@ func TestParseParameterList(t *testing.T) { parameters, err := parseParameterList(p, false) return parameters, err }, - Config{}, + Config{ + CommentTrackingEnabled: true, + }, ) } diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index 863500fe9e..4b8f435c97 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -113,6 +113,13 @@ func (l *lexer) Next() Token { endPos.column, ) + var comments ast.Comments + if len(l.currentComments) > 0 { + comments = ast.Comments{ + Leading: l.currentComments, + } + } + return Token{ Type: TokenEOF, Range: ast.NewRange( @@ -120,9 +127,7 @@ func (l *lexer) Next() Token { pos, pos, ), - Comments: ast.Comments{ - Leading: l.currentComments, - }, + Comments: comments, } } diff --git a/parser/parser_test.go b/parser/parser_test.go index a2a643c823..20abcba8ee 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1159,7 +1159,9 @@ fun second() {} /** Trailing multi-line comment of second */ `), - Config{}, + Config{ + CommentTrackingEnabled: true, + }, ) assert.Empty(t, errs) From 41b997264d6da06a0a334c8ac72b2d7514c99148 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Thu, 11 Dec 2025 12:35:12 +0100 Subject: [PATCH 80/84] ignore edge case func param comments for simplicity --- ast/comments.go | 4 ++++ parser/declaration_test.go | 2 +- parser/function.go | 27 +-------------------------- parser/lexer/lexer.go | 11 ++++++++--- 4 files changed, 14 insertions(+), 30 deletions(-) diff --git a/ast/comments.go b/ast/comments.go index ffb0d494c1..6cd029f584 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -76,6 +76,10 @@ var commentSuffixes = [][]byte{ blockCommentStringSuffix, } +func (c Comment) String() string { + return string(c.source) +} + // Text without opening/closing comment characters /*, /**, */, // func (c Comment) Text() []byte { withoutPrefixes := cutOptionalPrefixes(c.source, commentPrefixes) diff --git a/parser/declaration_test.go b/parser/declaration_test.go index c83a87730e..2ebbce9bc4 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -782,6 +782,7 @@ func TestParseParameterList(t *testing.T) { t.Parallel() + // Note: Some edge case comments are currently ignored (not mapped to the AST) for simplicity. result, errs := parse(`( /* Before param b */ a b : /* Before b type annotation */ Int /* After param b */, // Before param c c /* After c identifier */ : Int // After param c @@ -816,7 +817,6 @@ c /* After c identifier */ : Int // After param c StartPos: ast.Position{Line: 1, Column: 23, Offset: 23}, Comments: ast.Comments{ Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/* Before param b */")), ast.NewComment(nil, []byte("/* Before b type annotation */")), }, Trailing: []*ast.Comment{}, diff --git a/parser/function.go b/parser/function.go index 17b37ac3ea..5caaee27e9 100644 --- a/parser/function.go +++ b/parser/function.go @@ -38,7 +38,6 @@ func parsePurityAnnotation(p *parser) ast.FunctionPurity { func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterList, error) { var parameters []*ast.Parameter var endToken lexer.Token - var comments ast.Comments p.skipSpace() @@ -53,7 +52,6 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL p.next() expectParameter := true - var commaToken lexer.Token var atEnd bool progress := p.newProgress() @@ -72,7 +70,6 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL if err != nil { return nil, err } - parameter.Comments.Leading = append(parameter.Comments.Leading, commaToken.Comments.Trailing...) parameters = append(parameters, parameter) expectParameter = false @@ -84,7 +81,6 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } } // Skip the comma - commaToken = p.current p.next() expectParameter = true @@ -112,27 +108,6 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL } } - if len(parameters) == 0 { - comments.Leading = append( - comments.Leading, - startToken.Comments.All()..., - ) - } else { - comments.Leading = append( - comments.Leading, - startToken.Comments.Leading..., - ) - - var patched []*ast.Comment - patched = append(patched, startToken.Comments.Trailing...) - patched = append(patched, parameters[0].Comments.Leading...) - parameters[0].Comments.Leading = patched - } - comments.Trailing = append( - comments.Trailing, - endToken.Comments.All()..., - ) - return ast.NewParameterList( p.memoryGauge, parameters, @@ -141,7 +116,7 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL startToken.StartPos, endToken.EndPos, ), - comments, + ast.Comments{}, ), nil } diff --git a/parser/lexer/lexer.go b/parser/lexer/lexer.go index 4b8f435c97..1d0b67ae51 100644 --- a/parser/lexer/lexer.go +++ b/parser/lexer/lexer.go @@ -56,7 +56,7 @@ const ( lexerModeStringInterpolation ) -// trailingCommentsEndMarker is a sentinel value for determining the end of trailing comments +// trailingCommentsEndMarker is a sentinel value for determining the end of trailing comments (before the first newline). var trailingCommentsEndMarker *ast.Comment = nil type lexer struct { @@ -324,6 +324,8 @@ func (l *lexer) emit(ty TokenType, spaceOrError any, rangeStart ast.Position, co // tokenComments creates initial ast.Comments with leading comments, but empty trailing comments. // Trailing comments for the current token are determined when the next non-space token is consumed (in setPrevTokenTrailingComments). func (l *lexer) tokenComments(ty TokenType) ast.Comments { + // Do not attach comments to space tokens, since those are usually ignored during parsing, + // so mapping token comments to AST nodes would require extra effort. if ty == TokenSpace || !l.trackComments { return ast.EmptyComments } @@ -332,7 +334,7 @@ func (l *lexer) tokenComments(ty TokenType) ast.Comments { leadingComments := l.currentComments defer (func() { - // Mark as fully consumed + // Mark as fully consumed (clear by reallocating). l.currentComments = []*ast.Comment{} })() @@ -363,7 +365,8 @@ func (l *lexer) setPrevTokenTrailingComments() { // - leading comments of the next token var trailing, leading []*ast.Comment // If all previously seen tokens are spaces, - // we treat all comments as the leading set of the current token. + // treat all comments as the leading set for the current token, + // since comments shouldn't be attached to space tokens. isLeadingSet := latestNonSpaceTokenIndex == -1 for _, comment := range l.currentComments { if comment == trailingCommentsEndMarker { @@ -379,6 +382,8 @@ func (l *lexer) setPrevTokenTrailingComments() { l.currentComments = leading + // If there is a trailing set, attach it to the latest non-space token (if any), + // since comments shouldn't be attached to space tokens. if latestNonSpaceTokenIndex != -1 && len(trailing) > 0 { lastNonSpaceToken := &l.tokens[latestNonSpaceTokenIndex] if lastNonSpaceToken.Comments.Trailing == nil { From 9941b2e5e06dc0f0a4f7a9229b616c53fbe28996 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Thu, 11 Dec 2025 12:48:57 +0100 Subject: [PATCH 81/84] resolve all unit tests --- parser/function.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/parser/function.go b/parser/function.go index 5caaee27e9..68cda67b5a 100644 --- a/parser/function.go +++ b/parser/function.go @@ -116,7 +116,9 @@ func parseParameterList(p *parser, expectDefaultArguments bool) (*ast.ParameterL startToken.StartPos, endToken.EndPos, ), - ast.Comments{}, + ast.Comments{ + Trailing: endToken.Comments.Trailing, + }, ), nil } From 945779828996e3706cd40ebfc8058f29c06a2b8c Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Thu, 11 Dec 2025 12:56:57 +0100 Subject: [PATCH 82/84] move some test cases around --- parser/comment_test.go | 141 ------------------------------------ parser/declaration_test.go | 93 ++++++++++++++++++++++++ parser/parser_test.go | 93 ------------------------ parser/statement_test.go | 143 ++++++++++++++++++++++++++++++++++++- 4 files changed, 235 insertions(+), 235 deletions(-) diff --git a/parser/comment_test.go b/parser/comment_test.go index 5f67418667..b87590590d 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -200,144 +200,3 @@ func TestParseBlockComment(t *testing.T) { ) }) } - -func TestParseWhileStatementComment(t *testing.T) { - - t.Parallel() - - result, errs := testParseStatements(` -// before if -if true { - // noop -} // after if -// before else-if -else if true { - // noop -} -// before second else-if -else if true { - // noop -} /* after else-if */ else { - // noop -} // after else -`) - require.Empty(t, errs) - - AssertEqualWithDiff(t, - []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, - EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, - EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("// after if")), - }, - }, - }, - Else: &ast.Block{ - Statements: []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, - EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, - EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - }, - }, - }, - Else: &ast.Block{ - Statements: []ast.Statement{ - &ast.IfStatement{ - Test: &ast.BoolExpression{ - Value: true, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, - EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, - }, - }, - Then: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, - EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("/* after else-if */")), - }, - }, - }, - Else: &ast.Block{ - Statements: nil, - Range: ast.Range{ - StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - Comments: ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// noop")), - ast.NewComment(nil, []byte("// after else")), - }, - }, - }, - StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before second else-if")), - }, - }, - }, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - }, - StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before else-if")), - }, - }, - }, - }, - Range: ast.Range{ - StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, - EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, - }, - }, - StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, - Comments: ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("// before if")), - }, - }, - }, - }, - result, - ) -} diff --git a/parser/declaration_test.go b/parser/declaration_test.go index 2ebbce9bc4..ad6d9bc6cc 100644 --- a/parser/declaration_test.go +++ b/parser/declaration_test.go @@ -12509,3 +12509,96 @@ func TestParseTransactionDeclarationMissingOpeningBraceEOF(t *testing.T) { fixes[0].TextEdits[0].ApplyTo(code), ) } + +func TestParseDeclarationComments(t *testing.T) { + + t.Parallel() + + t.Run("event declaration", func(t *testing.T) { + res, errs := ParseDeclarations( + nil, + []byte(` +/// Before MyEvent +event MyEvent() // After MyEvent +/// Ignored +`), + Config{CommentTrackingEnabled: true}, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + event, ok := res[0].(*ast.CompositeDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, + }, event.Comments) + assert.Equal(t, " Before MyEvent", event.DeclarationDocString()) + + decl, ok := event.Members.Declarations()[0].(*ast.SpecialFunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, + }, decl.FunctionDeclaration.ParameterList.Comments) + assert.Equal(t, "", decl.DeclarationDocString()) + + }) + + t.Run("function declaration", func(t *testing.T) { + res, errs := ParseProgram( + nil, + []byte(` +/// Inline doc 1 of first +/// Inline doc 2 of first +fun first() {} // Trailing inline comment of first + +/** +Multi-line doc 1 of second +*/ +/** +Multi-line doc 2 of second +*/ +fun second() {} /** +Trailing multi-line comment of second +*/ +`), + Config{ + CommentTrackingEnabled: true, + }, + ) + + assert.Empty(t, errs) + assert.NotNil(t, res) + + first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/// Inline doc 1 of first")), + ast.NewComment(nil, []byte("/// Inline doc 2 of first")), + }, + }, first.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// Trailing inline comment of first")), + }, + }, first.FunctionBlock.Block.Comments) + + second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) + assert.True(t, ok) + assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) + assert.Equal(t, ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), + ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), + }, + }, second.Comments) + assert.Equal(t, ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), + }, + }, second.FunctionBlock.Block.Comments) + }) + +} diff --git a/parser/parser_test.go b/parser/parser_test.go index 20abcba8ee..08f5d1bb4c 100644 --- a/parser/parser_test.go +++ b/parser/parser_test.go @@ -1106,96 +1106,3 @@ func TestParseWhitespaceAtEnd(t *testing.T) { assert.Empty(t, errs) } - -func TestParseComments(t *testing.T) { - - t.Parallel() - - t.Run("event declaration", func(t *testing.T) { - res, errs := ParseDeclarations( - nil, - []byte(` -/// Before MyEvent -event MyEvent() // After MyEvent -/// Ignored -`), - Config{CommentTrackingEnabled: true}, - ) - - assert.Empty(t, errs) - assert.NotNil(t, res) - - event, ok := res[0].(*ast.CompositeDeclaration) - assert.True(t, ok) - assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{ast.NewComment(nil, []byte("/// Before MyEvent"))}, - }, event.Comments) - assert.Equal(t, " Before MyEvent", event.DeclarationDocString()) - - decl, ok := event.Members.Declarations()[0].(*ast.SpecialFunctionDeclaration) - assert.True(t, ok) - assert.Equal(t, ast.Comments{ - Trailing: []*ast.Comment{ast.NewComment(nil, []byte("// After MyEvent"))}, - }, decl.FunctionDeclaration.ParameterList.Comments) - assert.Equal(t, "", decl.DeclarationDocString()) - - }) - - t.Run("function declaration", func(t *testing.T) { - res, errs := ParseProgram( - nil, - []byte(` -/// Inline doc 1 of first -/// Inline doc 2 of first -fun first() {} // Trailing inline comment of first - -/** -Multi-line doc 1 of second -*/ -/** -Multi-line doc 2 of second -*/ -fun second() {} /** -Trailing multi-line comment of second -*/ -`), - Config{ - CommentTrackingEnabled: true, - }, - ) - - assert.Empty(t, errs) - assert.NotNil(t, res) - - first, ok := res.Declarations()[0].(*ast.FunctionDeclaration) - assert.True(t, ok) - assert.Equal(t, " Inline doc 1 of first\n Inline doc 2 of first", first.DeclarationDocString()) - assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/// Inline doc 1 of first")), - ast.NewComment(nil, []byte("/// Inline doc 2 of first")), - }, - }, first.Comments) - assert.Equal(t, ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("// Trailing inline comment of first")), - }, - }, first.FunctionBlock.Block.Comments) - - second, ok := res.Declarations()[1].(*ast.FunctionDeclaration) - assert.True(t, ok) - assert.Equal(t, "\nMulti-line doc 1 of second\n\n\nMulti-line doc 2 of second\n", second.DeclarationDocString()) - assert.Equal(t, ast.Comments{ - Leading: []*ast.Comment{ - ast.NewComment(nil, []byte("/**\nMulti-line doc 1 of second\n*/")), - ast.NewComment(nil, []byte("/**\nMulti-line doc 2 of second\n*/")), - }, - }, second.Comments) - assert.Equal(t, ast.Comments{ - Trailing: []*ast.Comment{ - ast.NewComment(nil, []byte("/**\nTrailing multi-line comment of second\n*/")), - }, - }, second.FunctionBlock.Block.Comments) - }) - -} diff --git a/parser/statement_test.go b/parser/statement_test.go index cbd76a9eff..717c9e2c7e 100644 --- a/parser/statement_test.go +++ b/parser/statement_test.go @@ -2445,11 +2445,152 @@ func TestParseIfStatementNoElse(t *testing.T) { ) } +func TestParseIfStatementWithComments(t *testing.T) { + + t.Parallel() + + result, errs := testParseStatements(` +// before if +if true { + // noop +} // after if +// before else-if +else if true { + // noop +} +// before second else-if +else if true { + // noop +} /* after else-if */ else { + // noop +} // after else +`) + require.Empty(t, errs) + + AssertEqualWithDiff(t, + []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 3, Offset: 17}, + EndPos: ast.Position{Line: 3, Column: 6, Offset: 20}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 3, Column: 8, Offset: 22}, + EndPos: ast.Position{Line: 5, Column: 0, Offset: 33}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after if")), + }, + }, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 8, Offset: 73}, + EndPos: ast.Position{Line: 7, Column: 11, Offset: 76}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 13, Offset: 78}, + EndPos: ast.Position{Line: 9, Column: 0, Offset: 89}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + }, + }, + }, + Else: &ast.Block{ + Statements: []ast.Statement{ + &ast.IfStatement{ + Test: &ast.BoolExpression{ + Value: true, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 8, Offset: 125}, + EndPos: ast.Position{Line: 11, Column: 11, Offset: 128}, + }, + }, + Then: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 13, Offset: 130}, + EndPos: ast.Position{Line: 13, Column: 0, Offset: 141}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("/* after else-if */")), + }, + }, + }, + Else: &ast.Block{ + Statements: nil, + Range: ast.Range{ + StartPos: ast.Position{Line: 13, Column: 27, Offset: 168}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + Comments: ast.Comments{ + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// noop")), + ast.NewComment(nil, []byte("// after else")), + }, + }, + }, + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before second else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 11, Column: 5, Offset: 122}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before else-if")), + }, + }, + }, + }, + Range: ast.Range{ + StartPos: ast.Position{Line: 7, Column: 5, Offset: 70}, + EndPos: ast.Position{Line: 15, Column: 0, Offset: 179}, + }, + }, + StartPos: ast.Position{Line: 3, Column: 0, Offset: 14}, + Comments: ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// before if")), + }, + }, + }, + }, + result, + ) +} + func TestParseWhileStatement(t *testing.T) { t.Parallel() - t.Run("while in function, comments", func(t *testing.T) { + t.Run("while in function", func(t *testing.T) { const code = ` fun test() { while true { From 43544d037e46d01083a21d28a0fd15b66e0500bc Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Thu, 11 Dec 2025 13:10:22 +0100 Subject: [PATCH 83/84] refactored to named `Comments Comments` struct field --- ast/access_test.go | 5 +++++ ast/attachment.go | 4 ++-- ast/attachment_test.go | 5 +++++ ast/block.go | 2 +- ast/block_test.go | 4 ++++ ast/composite.go | 4 ++-- ast/composite_test.go | 4 ++++ ast/entitlement_declaration.go | 4 ++-- ast/entitlement_declaration_test.go | 5 +++++ ast/expression.go | 4 ++-- ast/expression_test.go | 14 ++++++++++++++ ast/function_declaration.go | 2 +- ast/function_declaration_test.go | 14 ++++++++++++++ ast/import.go | 2 +- ast/import_test.go | 1 + ast/interface.go | 2 +- ast/interface_test.go | 3 +++ ast/parameter.go | 2 +- ast/statement.go | 16 ++++++++-------- ast/statement_test.go | 16 ++++++++++++++++ ast/transaction_declaration.go | 2 +- ast/transaction_declaration_test.go | 2 ++ ast/type.go | 2 +- ast/type_test.go | 22 ++++++++++++++++++++-- ast/variable_declaration.go | 2 +- ast/variable_declaration_test.go | 2 ++ 26 files changed, 119 insertions(+), 26 deletions(-) diff --git a/ast/access_test.go b/ast/access_test.go index 0ad41031ae..518169f50f 100644 --- a/ast/access_test.go +++ b/ast/access_test.go @@ -52,6 +52,7 @@ func TestMappedAccess_MarshalJSON(t *testing.T) { assert.JSONEq(t, `{ "EntitlementMap": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -82,6 +83,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { "ConjunctiveElements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -92,6 +94,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "F", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -115,6 +118,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { "DisjunctiveElements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -125,6 +129,7 @@ func TestEntitlementAccess_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "F", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/attachment.go b/ast/attachment.go index 6637006a1a..6a8aac9cf7 100644 --- a/ast/attachment.go +++ b/ast/attachment.go @@ -34,8 +34,8 @@ type AttachmentDeclaration struct { BaseType *NominalType Conformances []*NominalType Members *Members + Comments Comments Range - Comments } var _ Element = &AttachmentDeclaration{} @@ -292,7 +292,7 @@ type RemoveStatement struct { Attachment *NominalType Value Expression StartPos Position `json:"-"` - Comments + Comments Comments } var _ Element = &RemoveStatement{} diff --git a/ast/attachment_test.go b/ast/attachment_test.go index ffd7271f6a..4be880b94d 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -77,6 +77,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ` { "Type": "AttachmentDeclaration", + "Comments": {}, "Access": "AccessAll", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -87,6 +88,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { }, "BaseType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Bar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -99,6 +101,7 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Baz", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -473,6 +476,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { ` { "Type": "RemoveStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 3, "Line": 2, "Column": 5}, "Value": { @@ -488,6 +492,7 @@ func TestRemoveStatement_MarshallJSON(t *testing.T) { }, "Attachment": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "E", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/block.go b/ast/block.go index ad99a29226..c4fadd817f 100644 --- a/ast/block.go +++ b/ast/block.go @@ -28,8 +28,8 @@ import ( type Block struct { Statements []Statement + Comments Comments Range - Comments } var _ Element = &Block{} diff --git a/ast/block_test.go b/ast/block_test.go index 7d94c74537..1dcfed5e32 100644 --- a/ast/block_test.go +++ b/ast/block_test.go @@ -57,6 +57,7 @@ func TestBlock_MarshalJSON(t *testing.T) { ` { "Type": "Block", + "Comments": {}, "Statements": [ { "Type": "ExpressionStatement", @@ -225,6 +226,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [ { "Type": "ExpressionStatement", @@ -329,6 +331,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} @@ -356,6 +359,7 @@ func TestFunctionBlock_MarshalJSON(t *testing.T) { }, { "Type": "EmitCondition", + "Comments": {}, "InvocationExpression": { "Type": "InvocationExpression", "InvokedExpression": { diff --git a/ast/composite.go b/ast/composite.go index ee1f6ad447..bc080ddbab 100644 --- a/ast/composite.go +++ b/ast/composite.go @@ -56,7 +56,7 @@ type CompositeDeclaration struct { Range Access Access CompositeKind common.CompositeKind - Comments + Comments Comments } var _ Element = &CompositeDeclaration{} @@ -313,7 +313,7 @@ type FieldDeclaration struct { Access Access VariableKind VariableKind Flags FieldDeclarationFlags - Comments + Comments Comments } var _ Element = &FieldDeclaration{} diff --git a/ast/composite_test.go b/ast/composite_test.go index 926528d286..199262ee99 100644 --- a/ast/composite_test.go +++ b/ast/composite_test.go @@ -70,6 +70,7 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "FieldDeclaration", + "Comments": {}, "Access": "AccessAll", "IsStatic": true, "IsNative": true, @@ -83,6 +84,7 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -469,6 +471,7 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "CompositeDeclaration", + "Comments": {}, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -479,6 +482,7 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, diff --git a/ast/entitlement_declaration.go b/ast/entitlement_declaration.go index 93c596d1b5..dda3085459 100644 --- a/ast/entitlement_declaration.go +++ b/ast/entitlement_declaration.go @@ -31,8 +31,8 @@ import ( type EntitlementDeclaration struct { Access Access Identifier Identifier + Comments Comments Range - Comments } var _ Element = &EntitlementDeclaration{} @@ -169,8 +169,8 @@ type EntitlementMappingDeclaration struct { Access Access Identifier Identifier Elements []EntitlementMapElement + Comments Comments Range - Comments } var _ Element = &EntitlementMappingDeclaration{} diff --git a/ast/entitlement_declaration_test.go b/ast/entitlement_declaration_test.go index 2108a380d6..c0ecd14d9a 100644 --- a/ast/entitlement_declaration_test.go +++ b/ast/entitlement_declaration_test.go @@ -56,6 +56,7 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementDeclaration", + "Comments": {}, "Access": "AccessAll", "Identifier": { "Identifier": "AB", @@ -176,6 +177,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementMappingDeclaration", + "Comments": {}, "Access": "AccessAll", "Identifier": { "Identifier": "AB", @@ -185,6 +187,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { "Elements": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -196,6 +199,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { { "Input": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -206,6 +210,7 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { }, "Output": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Y", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, diff --git a/ast/expression.go b/ast/expression.go index 7ff3dc5b40..084f2de7f4 100644 --- a/ast/expression.go +++ b/ast/expression.go @@ -301,8 +301,8 @@ type IntegerExpression struct { Value *big.Int `json:"-"` PositiveLiteral []byte Range - Base int - Comments + Base int + Comments Comments } var _ Element = &IntegerExpression{} diff --git a/ast/expression_test.go b/ast/expression_test.go index 1139f653d4..fcb33f98fd 100644 --- a/ast/expression_test.go +++ b/ast/expression_test.go @@ -202,6 +202,7 @@ func TestIntegerExpression_MarshalJSON(t *testing.T) { ` { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "4_2", "Value": "42", "Base": 10, @@ -1966,6 +1967,7 @@ func TestUnaryExpression_MarshalJSON(t *testing.T) { "Operation": "OperationNegate", "Expression": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -2230,6 +2232,7 @@ func TestBinaryExpression_MarshalJSON(t *testing.T) { "Operation": "OperationPlus", "Left": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -2238,6 +2241,7 @@ func TestBinaryExpression_MarshalJSON(t *testing.T) { }, "Right": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "99", "Value": "99", "Base": 10, @@ -3254,6 +3258,7 @@ func TestConditionalExpression_MarshalJSON(t *testing.T) { }, "Then": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -3262,6 +3267,7 @@ func TestConditionalExpression_MarshalJSON(t *testing.T) { }, "Else": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "99", "Value": "99", "Base": 10, @@ -3851,6 +3857,7 @@ func TestInvocationExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -4329,6 +4336,7 @@ func TestCastingExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -4728,6 +4736,7 @@ func TestCreateExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -5169,8 +5178,10 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { { "Type": "FunctionExpression", "ParameterList": { + "Comments": {}, "Parameters": [ { + "Comments": {}, "Label": "ok", "Identifier": { "Identifier": "foobar", @@ -5181,6 +5192,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -5205,6 +5217,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -5220,6 +5233,7 @@ func TestFunctionExpression_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} diff --git a/ast/function_declaration.go b/ast/function_declaration.go index b481cfe8cc..3824986cb2 100644 --- a/ast/function_declaration.go +++ b/ast/function_declaration.go @@ -73,7 +73,7 @@ type FunctionDeclaration struct { StartPos Position `json:"-"` Access Access Flags FunctionDeclarationFlags - Comments + Comments Comments } var _ Element = &FunctionDeclaration{} diff --git a/ast/function_declaration_test.go b/ast/function_declaration_test.go index 87c7a067fe..1472cabe5a 100644 --- a/ast/function_declaration_test.go +++ b/ast/function_declaration_test.go @@ -128,6 +128,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { { "Type": "FunctionDeclaration", "Access": "AccessAll", + "Comments": {}, "IsStatic": true, "IsNative": true, "Identifier": { @@ -181,6 +182,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": { "Offset": 46, "Line": 47, @@ -220,9 +222,11 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { } }, "ParameterList": { + "Comments": {}, "Parameters": [ { "Label": "ok", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -232,6 +236,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -256,6 +261,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -271,6 +277,7 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} @@ -606,6 +613,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "FunctionDeclaration": { "Type": "FunctionDeclaration", "Access": "AccessNotSpecified", + "Comments": {}, "IsStatic": false, "IsNative": true, "Identifier": { @@ -659,6 +667,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": { "Offset": 46, "Line": 47, @@ -698,8 +707,10 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { } }, "ParameterList": { + "Comments": {}, "Parameters": [ { + "Comments": {}, "Label": "ok", "Identifier": { "Identifier": "foobar", @@ -710,6 +721,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { @@ -734,6 +746,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 22, "Line": 23, "Column": 24}, @@ -749,6 +762,7 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "Type": "FunctionBlock", "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 28, "Line": 29, "Column": 30}, "EndPos": {"Offset": 31, "Line": 32, "Column": 33} diff --git a/ast/import.go b/ast/import.go index 9b1218a38d..2499a8be16 100644 --- a/ast/import.go +++ b/ast/import.go @@ -42,7 +42,7 @@ type ImportDeclaration struct { Imports []Import Range LocationPos Position - Comments + Comments Comments } var _ Element = &ImportDeclaration{} diff --git a/ast/import_test.go b/ast/import_test.go index eb9869ae25..8db6cbdfe5 100644 --- a/ast/import_test.go +++ b/ast/import_test.go @@ -68,6 +68,7 @@ func TestImportDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "ImportDeclaration", + "Comments": {}, "Imports": [ { "Identifier": { diff --git a/ast/interface.go b/ast/interface.go index bd86a30f07..b7e98b87a3 100644 --- a/ast/interface.go +++ b/ast/interface.go @@ -35,7 +35,7 @@ type InterfaceDeclaration struct { Range Access Access CompositeKind common.CompositeKind - Comments + Comments Comments } var _ Element = &InterfaceDeclaration{} diff --git a/ast/interface_test.go b/ast/interface_test.go index 6a43e9c483..8d4490ad1c 100644 --- a/ast/interface_test.go +++ b/ast/interface_test.go @@ -62,6 +62,7 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", + "Comments": {}, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -118,6 +119,7 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", + "Comments": {}, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -128,6 +130,7 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { "Conformances": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, diff --git a/ast/parameter.go b/ast/parameter.go index 3d1eaef29c..74ed9433d0 100644 --- a/ast/parameter.go +++ b/ast/parameter.go @@ -32,7 +32,7 @@ type Parameter struct { Label string Identifier Identifier StartPos Position `json:"-"` - Comments + Comments Comments } func NewParameter( diff --git a/ast/statement.go b/ast/statement.go index 48980eb31e..b328ade3e0 100644 --- a/ast/statement.go +++ b/ast/statement.go @@ -38,8 +38,8 @@ type Statement interface { type ReturnStatement struct { Expression Expression + Comments Comments Range - Comments } var _ Element = &ReturnStatement{} @@ -103,8 +103,8 @@ func (s *ReturnStatement) MarshalJSON() ([]byte, error) { // BreakStatement type BreakStatement struct { + Comments Comments Range - Comments } var _ Element = &BreakStatement{} @@ -152,8 +152,8 @@ func (s *BreakStatement) MarshalJSON() ([]byte, error) { // ContinueStatement type ContinueStatement struct { + Comments Comments Range - Comments } var _ Element = &ContinueStatement{} @@ -323,7 +323,7 @@ type WhileStatement struct { Test Expression Block *Block StartPos Position `json:"-"` - Comments + Comments Comments } var _ Element = &WhileStatement{} @@ -402,7 +402,7 @@ type ForStatement struct { Block *Block Identifier Identifier StartPos Position `json:"-"` - Comments + Comments Comments } var _ Element = &ForStatement{} @@ -500,7 +500,7 @@ func (s *ForStatement) MarshalJSON() ([]byte, error) { type EmitStatement struct { InvocationExpression *InvocationExpression StartPos Position `json:"-"` - Comments + Comments Comments } var _ Element = &EmitStatement{} @@ -768,8 +768,8 @@ func (s *ExpressionStatement) String() string { type SwitchStatement struct { Expression Expression Cases []*SwitchCase + Comments Comments Range - Comments } var _ Element = &SwitchStatement{} @@ -867,8 +867,8 @@ func (s *SwitchStatement) MarshalJSON() ([]byte, error) { type SwitchCase struct { Expression Expression Statements []Statement + Comments Comments Range - Comments } func NewSwitchCase( diff --git a/ast/statement_test.go b/ast/statement_test.go index e1a61d244a..39b36b306e 100644 --- a/ast/statement_test.go +++ b/ast/statement_test.go @@ -156,6 +156,7 @@ func TestReturnStatement_MarshalJSON(t *testing.T) { ` { "Type": "ReturnStatement", + "Comments": {}, "Expression": { "Type": "BoolExpression", "Value": false, @@ -258,6 +259,7 @@ func TestBreakStatement_MarshalJSON(t *testing.T) { ` { "Type": "BreakStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} } @@ -305,6 +307,7 @@ func TestContinueStatement_MarshalJSON(t *testing.T) { ` { "Type": "ContinueStatement", + "Comments": {}, "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} } @@ -378,12 +381,14 @@ func TestIfStatement_MarshalJSON(t *testing.T) { }, "Then": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, "EndPos": {"Offset": 10, "Line": 11, "Column": 12} }, "Else": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 13, "Line": 14, "Column": 15}, "EndPos": {"Offset": 16, "Line": 17, "Column": 18} @@ -598,6 +603,7 @@ func TestWhileStatement_MarshalJSON(t *testing.T) { ` { "Type": "WhileStatement", + "Comments": {}, "Test": { "Type": "BoolExpression", "Value": false, @@ -606,6 +612,7 @@ func TestWhileStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, "EndPos": {"Offset": 10, "Line": 11, "Column": 12} @@ -738,6 +745,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { ` { "Type": "ForStatement", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -752,6 +760,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, "EndPos": {"Offset": 13, "Line": 14, "Column": 15} @@ -802,6 +811,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { ` { "Type": "ForStatement", + "Comments": {}, "Index": { "Identifier": "i", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -820,6 +830,7 @@ func TestForStatement_MarshalJSON(t *testing.T) { }, "Block": { "Type": "Block", + "Comments": {}, "Statements": [], "StartPos":{"Offset": 13, "Line": 14, "Column": 15}, "EndPos": {"Offset": 16, "Line": 17, "Column": 18} @@ -1346,6 +1357,7 @@ func TestEmitStatement_MarshalJSON(t *testing.T) { ` { "Type": "EmitStatement", + "Comments": {}, "InvocationExpression": { "Type": "InvocationExpression", "InvokedExpression": { @@ -1364,6 +1376,7 @@ func TestEmitStatement_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -1549,6 +1562,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { ` { "Type": "SwitchStatement", + "Comments": {}, "Expression": { "Type": "IdentifierExpression", "Comments": {}, @@ -1563,6 +1577,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { "Cases": [ { "Type": "SwitchCase", + "Comments": {}, "Expression": { "Type": "BoolExpression", "Value": false, @@ -1592,6 +1607,7 @@ func TestSwitchStatement_MarshalJSON(t *testing.T) { }, { "Type": "SwitchCase", + "Comments": {}, "Expression": null, "Statements": [ { diff --git a/ast/transaction_declaration.go b/ast/transaction_declaration.go index 655ad340fa..7355ef57cb 100644 --- a/ast/transaction_declaration.go +++ b/ast/transaction_declaration.go @@ -34,8 +34,8 @@ type TransactionDeclaration struct { PostConditions *Conditions DocString string Fields []*FieldDeclaration + Comments Comments Range - Comments } var _ Element = &TransactionDeclaration{} diff --git a/ast/transaction_declaration_test.go b/ast/transaction_declaration_test.go index 9381524649..05f85ac12e 100644 --- a/ast/transaction_declaration_test.go +++ b/ast/transaction_declaration_test.go @@ -75,7 +75,9 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "TransactionDeclaration", + "Comments": {}, "ParameterList": { + "Comments": {}, "Parameters": [], "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6} diff --git a/ast/type.go b/ast/type.go index 6d3c4bb750..0e54c7d8c0 100644 --- a/ast/type.go +++ b/ast/type.go @@ -111,7 +111,7 @@ func IsEmptyType(t Type) bool { type NominalType struct { NestedIdentifiers []Identifier `json:",omitempty"` Identifier Identifier - Comments + Comments Comments } var _ Type = &NominalType{} diff --git a/ast/type_test.go b/ast/type_test.go index a0cc161586..5f3c748409 100644 --- a/ast/type_test.go +++ b/ast/type_test.go @@ -187,6 +187,7 @@ func TestTypeAnnotation_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -330,6 +331,7 @@ func TestNominalType_MarshalJSON(t *testing.T) { ` { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -446,6 +448,7 @@ func TestOptionalType_MarshalJSON(t *testing.T) { "Type": "OptionalType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -579,6 +582,7 @@ func TestVariableSizedType_MarshalJSON(t *testing.T) { "Type": "VariableSizedType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -831,6 +835,7 @@ func TestConstantSizedType_MarshalJSON(t *testing.T) { "Type": "ConstantSizedType", "ElementType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "foobar", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -841,6 +846,7 @@ func TestConstantSizedType_MarshalJSON(t *testing.T) { }, "Size": { "Type": "IntegerExpression", + "Comments": {}, "PositiveLiteral": "42", "Value": "42", "Base": 10, @@ -1090,6 +1096,7 @@ func TestDictionaryType_MarshalJSON(t *testing.T) { "Type": "DictionaryType", "KeyType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1100,6 +1107,7 @@ func TestDictionaryType_MarshalJSON(t *testing.T) { }, "ValueType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -1331,6 +1339,7 @@ func TestFunctionType_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -1348,6 +1357,7 @@ func TestFunctionType_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, @@ -1858,8 +1868,9 @@ func TestReferenceType_MarshalJSON(t *testing.T) { "LegacyAuthorized": false, "Authorization": { "ConjunctiveElements": [ - { + { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "X", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -1868,8 +1879,9 @@ func TestReferenceType_MarshalJSON(t *testing.T) { "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, "EndPos": {"Offset": 0, "Line": 0, "Column": 0} }, - { + { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "Y", "StartPos": {"Offset": 0, "Line": 0, "Column": 0}, @@ -1882,6 +1894,7 @@ func TestReferenceType_MarshalJSON(t *testing.T) { }, "ReferencedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -2088,6 +2101,7 @@ func TestIntersectionType_MarshalJSON(t *testing.T) { "Types": [ { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -2098,6 +2112,7 @@ func TestIntersectionType_MarshalJSON(t *testing.T) { }, { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "EF", "StartPos": {"Offset": 7, "Line": 8, "Column": 9}, @@ -2402,6 +2417,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "Type": "InstantiationType", "InstantiatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "AB", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, @@ -2415,6 +2431,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "CD", "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, @@ -2430,6 +2447,7 @@ func TestInstantiationType_MarshalJSON(t *testing.T) { "IsResource": false, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "Identifier": { "Identifier": "EF", "StartPos": {"Offset": 10, "Line": 11, "Column": 12}, diff --git a/ast/variable_declaration.go b/ast/variable_declaration.go index 625481e982..ce14eea8fe 100644 --- a/ast/variable_declaration.go +++ b/ast/variable_declaration.go @@ -37,7 +37,7 @@ type VariableDeclaration struct { StartPos Position `json:"-"` Access Access IsConstant bool - Comments + Comments Comments } var _ Element = &VariableDeclaration{} diff --git a/ast/variable_declaration_test.go b/ast/variable_declaration_test.go index dd3172fa99..3be0c1203c 100644 --- a/ast/variable_declaration_test.go +++ b/ast/variable_declaration_test.go @@ -87,6 +87,7 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { { "Type": "VariableDeclaration", "Access": "AccessAll", + "Comments": {}, "IsConstant": true, "Identifier": { "Identifier": "foo", @@ -99,6 +100,7 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { "IsResource": true, "AnnotatedType": { "Type": "NominalType", + "Comments": {}, "StartPos": {"Offset": 4, "Line": 5, "Column": 6}, "EndPos": {"Offset": 5, "Line": 5, "Column": 7}, "Identifier": { From d1ded08b2358fafa55582f9d3ba767ae231e3f78 Mon Sep 17 00:00:00 2001 From: Bartolomej Kozorog Date: Fri, 12 Dec 2025 13:20:05 +0100 Subject: [PATCH 84/84] fix comments json serialization --- ast/attachment_test.go | 4 ++- ast/comments.go | 28 ++++++++++++++++-- ast/composite_test.go | 8 +++-- ast/entitlement_declaration_test.go | 8 +++-- ast/function_declaration_test.go | 8 +++-- ast/interface_test.go | 8 +++-- ast/transaction_declaration_test.go | 4 ++- ast/variable_declaration_test.go | 4 ++- parser/comment_test.go | 45 +++++++++++++++++++++++++++++ 9 files changed, 104 insertions(+), 13 deletions(-) diff --git a/ast/attachment_test.go b/ast/attachment_test.go index 4be880b94d..d19701bddf 100644 --- a/ast/attachment_test.go +++ b/ast/attachment_test.go @@ -77,7 +77,9 @@ func TestAttachmentDeclaration_MarshallJSON(t *testing.T) { ` { "Type": "AttachmentDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "StartPos": {"Offset": 1, "Line": 2, "Column": 3}, "EndPos": {"Offset": 4, "Line": 5, "Column": 6}, diff --git a/ast/comments.go b/ast/comments.go index 6cd029f584..a3ff567acc 100644 --- a/ast/comments.go +++ b/ast/comments.go @@ -2,14 +2,15 @@ package ast import ( "bytes" + "encoding/json" "strings" "github.com/onflow/cadence/common" ) type Comments struct { - Leading []*Comment `json:"-"` - Trailing []*Comment `json:"-"` + Leading []*Comment + Trailing []*Comment } var EmptyComments = Comments{} @@ -103,3 +104,26 @@ func cutOptionalSuffixes(input []byte, suffixes [][]byte) (output []byte) { } return } + +func (c Comments) MarshalJSON() ([]byte, error) { + cj := struct { + Leading []string `json:"Leading,omitempty"` + Trailing []string `json:"Trailing,omitempty"` + }{} + + if len(c.Leading) > 0 { + cj.Leading = make([]string, len(c.Leading)) + for i, comment := range c.Leading { + cj.Leading[i] = comment.String() + } + } + + if len(c.Trailing) > 0 { + cj.Trailing = make([]string, len(c.Trailing)) + for i, comment := range c.Trailing { + cj.Trailing[i] = comment.String() + } + } + + return json.Marshal(cj) +} diff --git a/ast/composite_test.go b/ast/composite_test.go index 199262ee99..671cc901e2 100644 --- a/ast/composite_test.go +++ b/ast/composite_test.go @@ -70,7 +70,9 @@ func TestFieldDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "FieldDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "IsStatic": true, "IsNative": true, @@ -471,7 +473,9 @@ func TestCompositeDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "CompositeDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { diff --git a/ast/entitlement_declaration_test.go b/ast/entitlement_declaration_test.go index c0ecd14d9a..85d72e114c 100644 --- a/ast/entitlement_declaration_test.go +++ b/ast/entitlement_declaration_test.go @@ -56,7 +56,9 @@ func TestEntitlementDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "Identifier": { "Identifier": "AB", @@ -177,7 +179,9 @@ func TestEntitlementMappingDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "EntitlementMappingDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "Identifier": { "Identifier": "AB", diff --git a/ast/function_declaration_test.go b/ast/function_declaration_test.go index 1472cabe5a..64ad3cf666 100644 --- a/ast/function_declaration_test.go +++ b/ast/function_declaration_test.go @@ -128,7 +128,9 @@ func TestFunctionDeclaration_MarshalJSON(t *testing.T) { { "Type": "FunctionDeclaration", "Access": "AccessAll", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "IsStatic": true, "IsNative": true, "Identifier": { @@ -613,7 +615,9 @@ func TestSpecialFunctionDeclaration_MarshalJSON(t *testing.T) { "FunctionDeclaration": { "Type": "FunctionDeclaration", "Access": "AccessNotSpecified", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "IsStatic": false, "IsNative": true, "Identifier": { diff --git a/ast/interface_test.go b/ast/interface_test.go index 8d4490ad1c..41f2e50d89 100644 --- a/ast/interface_test.go +++ b/ast/interface_test.go @@ -62,7 +62,9 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { @@ -119,7 +121,9 @@ func TestInterfaceDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "InterfaceDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "Access": "AccessAll", "CompositeKind": "CompositeKindResource", "Identifier": { diff --git a/ast/transaction_declaration_test.go b/ast/transaction_declaration_test.go index 05f85ac12e..061eb71730 100644 --- a/ast/transaction_declaration_test.go +++ b/ast/transaction_declaration_test.go @@ -75,7 +75,9 @@ func TestTransactionDeclaration_MarshalJSON(t *testing.T) { ` { "Type": "TransactionDeclaration", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "ParameterList": { "Comments": {}, "Parameters": [], diff --git a/ast/variable_declaration_test.go b/ast/variable_declaration_test.go index 3be0c1203c..a971a91dbc 100644 --- a/ast/variable_declaration_test.go +++ b/ast/variable_declaration_test.go @@ -87,7 +87,9 @@ func TestVariableDeclaration_MarshalJSON(t *testing.T) { { "Type": "VariableDeclaration", "Access": "AccessAll", - "Comments": {}, + "Comments": { + "Leading": ["///test"] + }, "IsConstant": true, "Identifier": { "Identifier": "foo", diff --git a/parser/comment_test.go b/parser/comment_test.go index b87590590d..4f1230d4fd 100644 --- a/parser/comment_test.go +++ b/parser/comment_test.go @@ -19,9 +19,11 @@ package parser import ( + "encoding/json" "math/big" "testing" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/onflow/cadence/errors" @@ -200,3 +202,46 @@ func TestParseBlockComment(t *testing.T) { ) }) } + +func TestCommentsJSONSerialization(t *testing.T) { + t.Parallel() + + t.Run("comments with leading and trailing", func(t *testing.T) { + t.Parallel() + + comments := ast.Comments{ + Leading: []*ast.Comment{ + ast.NewComment(nil, []byte("// leading comment 1")), + ast.NewComment(nil, []byte("/* leading comment 2 */")), + }, + Trailing: []*ast.Comment{ + ast.NewComment(nil, []byte("// trailing comment")), + }, + } + + jsonBytes, err := json.Marshal(comments) + require.NoError(t, err) + + assert.JSONEq(t, + `{ + "Leading": ["// leading comment 1", "/* leading comment 2 */"], + "Trailing": ["// trailing comment"] + }`, + string(jsonBytes), + ) + }) + + t.Run("empty comments", func(t *testing.T) { + t.Parallel() + + comments := ast.Comments{} + + jsonBytes, err := json.Marshal(comments) + require.NoError(t, err) + + assert.JSONEq(t, + `{}`, + string(jsonBytes), + ) + }) +}