diff --git a/Sources/SwiftParser/Declarations.swift b/Sources/SwiftParser/Declarations.swift index 6735f44fd74..68867cfca87 100644 --- a/Sources/SwiftParser/Declarations.swift +++ b/Sources/SwiftParser/Declarations.swift @@ -407,22 +407,6 @@ extension Parser { } extension Parser { - /// Attempt to consume an ellipsis prefix, splitting the current token if - /// necessary. - mutating func tryConsumeEllipsisPrefix() -> RawTokenSyntax? { - // It is not sufficient to check currentToken.isEllipsis here, as we may - // have something like '...>'. - // TODO: Recovery for different numbers of dots (which also needs to be - // done for regular variadics). - guard self.at(anyIn: Operator.self) != nil else { return nil } - let text = self.currentToken.tokenText - guard text.hasPrefix("...") else { return nil } - return self.consumePrefix( - SyntaxText(rebasing: text.prefix(3)), - as: .ellipsis - ) - } - mutating func parseGenericParameters() -> RawGenericParameterClauseSyntax { if let remainingTokens = remainingTokensIfMaximumNestingLevelReached() { return RawGenericParameterClauseSyntax( @@ -435,12 +419,7 @@ extension Parser { ) } - let langle: RawTokenSyntax - if self.currentToken.starts(with: "<") { - langle = self.consumePrefix("<", as: .leftAngle) - } else { - langle = missingToken(.leftAngle) - } + let langle = self.expectWithoutRecovery(prefix: "<", as: .leftAngle) var elements = [RawGenericParameterSyntax]() do { var keepGoing: RawTokenSyntax? = nil @@ -452,14 +431,13 @@ extension Parser { var each = self.consume(if: .keyword(.each)) let (unexpectedBetweenEachAndName, name) = self.expectIdentifier(allowSelfOrCapitalSelfAsIdentifier: true) - if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty && !self.currentToken.starts(with: ">") - { + if attributes == nil && each == nil && unexpectedBetweenEachAndName == nil && name.isMissing && elements.isEmpty && !self.at(prefix: ">") { break } // Parse the unsupported ellipsis for a type parameter pack 'T...'. let unexpectedBetweenNameAndColon: RawUnexpectedNodesSyntax? - if let ellipsis = tryConsumeEllipsisPrefix() { + if let ellipsis = self.consume(ifPrefix: "...", as: .ellipsis) { unexpectedBetweenNameAndColon = RawUnexpectedNodesSyntax([ellipsis], arena: self.arena) if each == nil { each = missingToken(.each) @@ -518,12 +496,7 @@ extension Parser { whereClause = nil } - let rangle: RawTokenSyntax - if self.currentToken.starts(with: ">") { - rangle = self.consumePrefix(">", as: .rightAngle) - } else { - rangle = RawTokenSyntax(missing: .rightAngle, arena: self.arena) - } + let rangle = expectWithoutRecovery(prefix: ">", as: .rightAngle) let parameters: RawGenericParameterListSyntax if elements.isEmpty && rangle.isMissing { @@ -934,7 +907,7 @@ extension Parser { } // Detect an attempt to use (early syntax) type parameter pack. - let ellipsis = tryConsumeEllipsisPrefix() + let ellipsis = self.consume(ifPrefix: "...", as: .ellipsis) // Parse optional inheritance clause. let inheritance: RawTypeInheritanceClauseSyntax? @@ -1011,7 +984,7 @@ extension Parser { } let generics: RawGenericParameterClauseSyntax? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { generics = self.parseGenericParameters() } else { generics = nil @@ -1145,7 +1118,7 @@ extension Parser { } let genericParams: RawGenericParameterClauseSyntax? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { genericParams = self.parseGenericParameters() } else { genericParams = nil @@ -1229,14 +1202,14 @@ extension Parser { let (unexpectedBeforeSubscriptKeyword, subscriptKeyword) = self.eat(handle) let unexpectedName: RawTokenSyntax? - if self.at(.identifier) && self.peek().starts(with: "<") || self.peek().rawTokenKind == .leftParen { + if self.at(.identifier) && self.peek().tokenText.hasPrefix("<") || self.peek().rawTokenKind == .leftParen { unexpectedName = self.consumeAnyToken() } else { unexpectedName = nil } let genericParameterClause: RawGenericParameterClauseSyntax? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { genericParameterClause = self.parseGenericParameters() } else { genericParameterClause = nil @@ -1617,7 +1590,7 @@ extension Parser { // Parse a generic parameter list if it is present. let generics: RawGenericParameterClauseSyntax? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { generics = self.parseGenericParameters() } else { generics = nil @@ -2012,7 +1985,7 @@ extension Parser { // Optional generic parameters. let genericParams: RawGenericParameterClauseSyntax? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { genericParams = self.parseGenericParameters() } else { genericParams = nil diff --git a/Sources/SwiftParser/Expressions.swift b/Sources/SwiftParser/Expressions.swift index 359c4efbe8c..72f87b8bc48 100644 --- a/Sources/SwiftParser/Expressions.swift +++ b/Sources/SwiftParser/Expressions.swift @@ -1023,21 +1023,12 @@ extension Parser { for _ in 0.. ... -> consumePrefix() -> [ ] ... mutating func consumePrefix(_ prefix: SyntaxText, as tokenKind: RawTokenKind) { + precondition( + tokenKind.defaultText == nil || prefix == tokenKind.defaultText!, + "If tokenKind has a defaultText, the prefix needs to match it" + ) let tokenText = self.currentToken.tokenText if tokenText == prefix { diff --git a/Sources/SwiftParser/Names.swift b/Sources/SwiftParser/Names.swift index 12fdf0f1fff..675aa3aca9c 100644 --- a/Sources/SwiftParser/Names.swift +++ b/Sources/SwiftParser/Names.swift @@ -228,7 +228,7 @@ extension Parser.Lookahead { guard lookahead.canParseTypeIdentifier() else { return false } - return lookahead.currentToken.starts(with: ".") + return lookahead.at(prefix: ".") } } @@ -289,12 +289,4 @@ extension Lexer.Lexeme { // constructed from them. return self.rawTokenKind == .keyword } - - func starts(with symbol: SyntaxText) -> Bool { - guard Operator(lexeme: self) != nil || self.rawTokenKind.isPunctuation else { - return false - } - - return self.tokenText.hasPrefix(symbol) - } } diff --git a/Sources/SwiftParser/Nominals.swift b/Sources/SwiftParser/Nominals.swift index 0757486a878..1a95c7a4afe 100644 --- a/Sources/SwiftParser/Nominals.swift +++ b/Sources/SwiftParser/Nominals.swift @@ -233,7 +233,7 @@ extension Parser { } let primaryOrGenerics: T.PrimaryOrGenerics? - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { primaryOrGenerics = T.parsePrimaryOrGenerics(&self) } else { primaryOrGenerics = nil @@ -352,18 +352,10 @@ extension Parser { ) } while keepGoing != nil && loopProgress.evaluate(currentToken) } - let unexpectedBeforeRangle: RawUnexpectedNodesSyntax? - let rangle: RawTokenSyntax - if self.currentToken.starts(with: ">") { - unexpectedBeforeRangle = nil - rangle = self.consumePrefix(">", as: .rightAngle) - } else { - (unexpectedBeforeRangle, rangle) = self.expect(.rightAngle) - } + let rangle = self.expectWithoutRecovery(prefix: ">", as: .rightAngle) return RawPrimaryAssociatedTypeClauseSyntax( leftAngle: langle, primaryAssociatedTypeList: RawPrimaryAssociatedTypeListSyntax(elements: associatedTypes, arena: self.arena), - unexpectedBeforeRangle, rightAngle: rangle, arena: self.arena ) diff --git a/Sources/SwiftParser/Parser.swift b/Sources/SwiftParser/Parser.swift index ab189f74c4a..bae1cbd082b 100644 --- a/Sources/SwiftParser/Parser.swift +++ b/Sources/SwiftParser/Parser.swift @@ -471,6 +471,17 @@ extension Parser { ) } + /// If the current token starts with the given prefix, consume the prefis as the given token kind. + /// + /// Otherwise, synthesize a missing token of the given kind. + mutating func expectWithoutRecovery(prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token { + if self.at(prefix: prefix) { + return consumePrefix(prefix, as: tokenKind) + } else { + return missingToken(tokenKind) + } + } + /// - Parameters: /// - keywordRecovery: If set to `true` and the parser is currently /// positioned at a keyword instead of an identifier, this method recovers @@ -594,6 +605,10 @@ extension Parser { _ prefix: SyntaxText, as tokenKind: RawTokenKind ) -> RawTokenSyntax { + precondition( + tokenKind.defaultText == nil || prefix == tokenKind.defaultText!, + "If tokenKind has a defaultText, the prefix needs to match it" + ) let current = self.currentToken // Current token can be either one-character token we want to consume... let tokenText = current.tokenText diff --git a/Sources/SwiftParser/TokenConsumer.swift b/Sources/SwiftParser/TokenConsumer.swift index d111d6942fd..e3e341e0b11 100644 --- a/Sources/SwiftParser/TokenConsumer.swift +++ b/Sources/SwiftParser/TokenConsumer.swift @@ -23,6 +23,13 @@ protocol TokenConsumer { /// Consume the current token and change its token kind to `remappedTokenKind`. mutating func consumeAnyToken(remapping remappedTokenKind: RawTokenKind) -> Token + /// Consumes a given token, or splits the current token into a leading token + /// matching the given token and a trailing token and consumes the leading + /// token. + /// + /// ... -> consumePrefix() -> [ ] ... + mutating func consumePrefix(_ prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token + /// Synthesize a missing token with `kind`. /// If `text` is not `nil`, use it for the token's text, otherwise use the token's default text. mutating func missingToken(_ kind: RawTokenKind, text: SyntaxText?) -> Token @@ -137,6 +144,12 @@ extension TokenConsumer { return nil } + /// Whether the current token’s text starts with the given prefix. + @inline(__always) + mutating func at(prefix: SyntaxText) -> Bool { + return self.currentToken.tokenText.hasPrefix(prefix) + } + /// Eat a token that we know we are currently positioned at, based on `at(anyIn:)`. @inline(__always) mutating func eat(_ handle: TokenConsumptionHandle) -> Token { @@ -248,6 +261,17 @@ extension TokenConsumer { return nil } } + + /// If the current token starts with the given prefix, consume the prefis as the given token kind. + /// + /// Otherwise, return `nil`. + mutating func consume(ifPrefix prefix: SyntaxText, as tokenKind: RawTokenKind) -> Token? { + if self.at(prefix: prefix) { + return consumePrefix(prefix, as: tokenKind) + } else { + return nil + } + } } // MARK: Convenience functions diff --git a/Sources/SwiftParser/Types.swift b/Sources/SwiftParser/Types.swift index c76b4bae59c..58716ee8746 100644 --- a/Sources/SwiftParser/Types.swift +++ b/Sources/SwiftParser/Types.swift @@ -422,8 +422,7 @@ extension Parser { /// generic-argument-list → generic-argument | generic-argument ',' generic-argument-list /// generic-argument → type mutating func parseGenericArguments() -> RawGenericArgumentClauseSyntax { - precondition(self.currentToken.starts(with: "<")) - let langle = self.consumePrefix("<", as: .leftAngle) + let langle = self.expectWithoutRecovery(prefix: "<", as: .leftAngle) var arguments = [RawGenericArgumentSyntax]() do { var keepGoing: RawTokenSyntax? = nil @@ -444,12 +443,7 @@ extension Parser { } while keepGoing != nil && loopProgress.evaluate(currentToken) } - let rangle: RawTokenSyntax - if self.currentToken.starts(with: ">") { - rangle = self.consumePrefix(">", as: .rightAngle) - } else { - rangle = RawTokenSyntax(missing: .rightAngle, arena: self.arena) - } + let rangle = self.expectWithoutRecovery(prefix: ">", as: .rightAngle) let args: RawGenericArgumentListSyntax if arguments.isEmpty && rangle.isMissing { @@ -656,7 +650,7 @@ extension Parser.Lookahead { return false } - if self.currentToken.isEllipsis { + if self.atContextualPunctuator("...") { self.consumeAnyToken() } @@ -884,7 +878,7 @@ extension Parser.Lookahead { self.consumeAnyToken() // Parse an optional generic argument list. - if self.currentToken.starts(with: "<") && !self.consumeGenericArguments() { + if self.at(prefix: "<") && !self.consumeGenericArguments() { return false } @@ -905,13 +899,11 @@ extension Parser.Lookahead { mutating func consumeGenericArguments() -> Bool { // Parse the opening '<'. - guard self.currentToken.starts(with: "<") else { + guard self.consume(ifPrefix: "<", as: .leftAngle) != nil else { return false } - self.consumePrefix("<", as: .leftAngle) - - if !self.currentToken.starts(with: ">") { + if !self.at(prefix: ">") { var loopProgress = LoopProgressCondition() repeat { guard self.canParseType() else { @@ -921,11 +913,10 @@ extension Parser.Lookahead { } while self.consume(if: .comma) != nil && loopProgress.evaluate(currentToken) } - guard self.currentToken.starts(with: ">") else { + guard self.consume(ifPrefix: ">", as: .rightAngle) != nil else { return false } - self.consumePrefix(">", as: .rightAngle) return true } } @@ -1001,7 +992,7 @@ extension Parser { extension Parser { mutating func parseResultType() -> RawTypeSyntax { - if self.currentToken.starts(with: "<") { + if self.at(prefix: "<") { let generics = self.parseGenericParameters() let baseType = self.parseType() return RawTypeSyntax( @@ -1076,10 +1067,6 @@ extension Lexer.Lexeme { || self.rawTokenKind == .prefixOperator } - var isEllipsis: Bool { - return self.isAnyOperator && self.tokenText == "..." - } - var isGenericTypeDisambiguatingToken: Bool { switch self.rawTokenKind { case .rightParen,