diff --git a/Changelog.md b/Changelog.md index 49bab269cb0..9f8a6373dbd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,6 +5,9 @@ Note: This is in reverse chronological order, so newer entries are added to the ## `main` * To clarify that the edits passed to `IncrementalParseTransition` are applied concurrently, introduce a new `ConcurrentEdit` type that provides the guarantee and allows translation of sequentially applied edits to the expected concurrent form. +* The `SwiftSyntaxParser` type and a few related types now live in their own module (also named `SwiftSyntaxParser`). This allows using `SwiftSyntax` for code generation purposes without having a compatible `_InternalSwiftSyntaxParser.dylib` around. + + `import SwiftSyntaxParser` where necessary. ## Swift 5.3 diff --git a/Package.swift b/Package.swift index 161170233a7..bb780274d0f 100644 --- a/Package.swift +++ b/Package.swift @@ -7,11 +7,13 @@ let package = Package( name: "SwiftSyntax", targets: [ .target(name: "_CSwiftSyntax"), - .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"], exclude: ["Inputs"]), + .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"]), .target(name: "SwiftSyntaxBuilder", dependencies: ["SwiftSyntax"]), .testTarget(name: "SwiftSyntaxBuilderTest", dependencies: ["SwiftSyntaxBuilder"]), - .target(name: "lit-test-helper", dependencies: ["SwiftSyntax"]), - .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax"]) + .target(name: "SwiftSyntaxParser", dependencies: ["SwiftSyntax"]), + .testTarget(name: "SwiftSyntaxParserTest", dependencies: ["SwiftSyntaxParser"], exclude: ["Inputs"]), + .target(name: "lit-test-helper", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]), + .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]) // Also see targets added below ] ) @@ -52,4 +54,5 @@ if ProcessInfo.processInfo.environment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] != nil } package.products.append(.library(name: "SwiftSyntax", type: libraryType, targets: ["SwiftSyntax"])) +package.products.append(.library(name: "SwiftSyntaxParser", type: libraryType, targets: ["SwiftSyntaxParser"])) package.products.append(.library(name: "SwiftSyntaxBuilder", type: libraryType, targets: ["SwiftSyntaxBuilder"])) diff --git a/Sources/SwiftSyntax/CNodes.swift b/Sources/SwiftSyntax/CNodes.swift new file mode 100644 index 00000000000..5408a46dbcc --- /dev/null +++ b/Sources/SwiftSyntax/CNodes.swift @@ -0,0 +1,72 @@ +//===---------------------------- CNodes.swift ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _CSwiftSyntax + +// These two types need to be exposed publicly, so they can't rely on the C +// types defined in _CSwiftSyntax. +public typealias CSyntaxKind = UInt16 +public typealias CClientNode = UnsafeMutableRawPointer + +typealias CSyntaxNode = swiftparse_syntax_node_t +typealias CSyntaxNodePtr = UnsafePointer +typealias CTokenKind = swiftparse_token_kind_t +typealias CTriviaKind = swiftparse_trivia_kind_t +typealias CTokenData = swiftparse_token_data_t +typealias CLayoutData = swiftparse_layout_data_t +typealias CTriviaPiecePtr = UnsafePointer +typealias CTriviaPiece = swiftparse_trivia_piece_t + +/// Computes a hash value that describes the layout of all C nodes which are +/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`. +/// This should match the value returned by the `cNodeLayoutHash` function in +/// the `SwiftSyntaxParser` module. +public func cNodeLayoutHash() -> Int { + var hasher = Hasher() + + // These two types are not defined in terms of the C types in SwiftSyntax. + // Let's include them specifically in the hash as well. + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.offset)) + hasher.combine(MemoryLayout.offset(of: \.length)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.length)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia)) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.range)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.nodes)) + hasher.combine(MemoryLayout.offset(of: \.nodes_count)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.present)) + + return hasher.finalize() +} diff --git a/Sources/SwiftSyntax/IncrementalParseTransition.swift b/Sources/SwiftSyntax/IncrementalParseTransition.swift index 0662275ecfc..624df35305e 100644 --- a/Sources/SwiftSyntax/IncrementalParseTransition.swift +++ b/Sources/SwiftSyntax/IncrementalParseTransition.swift @@ -199,11 +199,11 @@ public struct ConcurrentEdits { /// Provides a mechanism for the parser to skip regions of an incrementally /// updated source that was already parsed during a previous parse invocation. -internal struct IncrementalParseLookup { +public struct IncrementalParseLookup { fileprivate let transition: IncrementalParseTransition fileprivate var cursor: SyntaxCursor - init(transition: IncrementalParseTransition) { + public init(transition: IncrementalParseTransition) { self.transition = transition self.cursor = .init(root: transition.previousTree.data.absoluteRaw) } @@ -224,16 +224,16 @@ internal struct IncrementalParseLookup { /// /// - Parameters: /// - offset: The byte offset of the source string that is currently parsed. - /// - kind: The `SyntaxKind` that the parser expects at this position. + /// - kind: The `CSyntaxKind` that the parser expects at this position. /// - Returns: A `SyntaxNode` node from the previous parse invocation, /// representing the contents of this region, if it is still valid /// to re-use. `nil` otherwise. - mutating func lookUp(_ newOffset: Int, kind: SyntaxKind) -> SyntaxNode? { + public mutating func lookUp(_ newOffset: Int, kind: CSyntaxKind) -> SyntaxNode? { guard let prevOffset = translateToPreEditOffset(newOffset) else { return nil } let prevPosition = AbsolutePosition(utf8Offset: prevOffset) - let node = cursorLookup(prevPosition: prevPosition, kind: kind) + let node = cursorLookup(prevPosition: prevPosition, kind: .fromRawValue(kind)) if let delegate = reusedDelegate, let node = node { delegate.parserReusedNode( range: ByteSourceRange(offset: newOffset, length: node.byteSize), diff --git a/Sources/SwiftSyntax/Misc.swift.gyb b/Sources/SwiftSyntax/Misc.swift.gyb index 15be1f7c769..110f5ec8aa0 100644 --- a/Sources/SwiftSyntax/Misc.swift.gyb +++ b/Sources/SwiftSyntax/Misc.swift.gyb @@ -19,8 +19,6 @@ // //===----------------------------------------------------------------------===// -@_implementationOnly import _InternalSwiftSyntaxParser - extension SyntaxNode { public var isUnknown: Bool { return raw.kind.isUnknown } public var asUnknown: UnknownSyntax? { @@ -63,10 +61,3 @@ extension Syntax { } } } - -extension SyntaxParser { - static func verifyNodeDeclarationHash() -> Bool { - return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == - "${calculate_node_hash()}" - } -} diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index e6a738f444d..ff6f78d56a4 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -1216,12 +1216,12 @@ final class RawSyntax: ManagedBuffer { } extension RawSyntax { + static func moveFromOpaque(_ cn: CClientNode) -> RawSyntax { + return Unmanaged.fromOpaque(cn).takeRetainedValue() + } + static func moveFromOpaque(_ cn: CClientNode?) -> RawSyntax? { - if let subnode = cn { - return Unmanaged.fromOpaque(subnode).takeRetainedValue() - } else { - return nil - } + return cn.map(moveFromOpaque) } static func getFromOpaque(_ cn: CClientNode?) -> RawSyntax? { diff --git a/Sources/SwiftSyntax/_SyntaxParserInterop.swift b/Sources/SwiftSyntax/_SyntaxParserInterop.swift new file mode 100644 index 00000000000..6dd40231d21 --- /dev/null +++ b/Sources/SwiftSyntax/_SyntaxParserInterop.swift @@ -0,0 +1,54 @@ +//===--------------------- _SyntaxParserInterop.swift ---------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Namespace for methods needed by the `SwiftSyntaxParser` module to +/// efficiently create `SwiftSyntax` nodes from the C nodes created by the +/// parser. +public enum _SyntaxParserInterop { + private static func getRetainedOpaque(rawSyntax: RawSyntax) -> CClientNode { + return Unmanaged.passRetained(rawSyntax).toOpaque() + } + + /// Create a `RawSyntax` node for the given `cnode` and return an opaque + /// pointer to the `RawSyntax` node (a `CClientNode`). + /// After this method finishes, the `RawSyntax` node has a retain count of 1 + /// and is owned by whoever manages the returned `CClientNode`. Passing the + /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership + /// back to `SwiftSyntax`. + public static func getRetainedOpaqueRawSyntax( + cnode: UnsafeRawPointer, source: String + ) -> CClientNode { + let cnode = cnode.assumingMemoryBound(to: CSyntaxNode.self) + // Transfer ownership of the object to the C parser. We get ownership back + // via `moveFromCRawNode()`. + let node = RawSyntax.create(from: cnode, source: source) + return getRetainedOpaque(rawSyntax: node) + } + + /// Return an opaque pointer to the given `node`. + /// After this method finishes, the `RawSyntax` node has a retain count of 1 + /// and is owned by whoever manages the returned `CClientNode`. Passing the + /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership + /// back to `SwiftSyntax`. + public static func getRetainedOpaqueRawSyntax(node: SyntaxNode) + -> CClientNode { + return getRetainedOpaque(rawSyntax: node.raw) + } + + /// After an opaque pointer to a `RawSyntax` node has been created using one + /// of the methods above, transfer its ownership back to a `Syntax` node, + /// which is managed by `SwiftSyntax`. + public static func nodeFromRetainedOpaqueRawSyntax(_ cRoot: CClientNode) + -> Syntax { + return Syntax(SyntaxData.forRoot(RawSyntax.moveFromOpaque(cRoot))) + } +} diff --git a/Sources/SwiftSyntax/gyb_generated/Misc.swift b/Sources/SwiftSyntax/gyb_generated/Misc.swift index 2960af8142e..e2d633039c7 100644 --- a/Sources/SwiftSyntax/gyb_generated/Misc.swift +++ b/Sources/SwiftSyntax/gyb_generated/Misc.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -@_implementationOnly import _InternalSwiftSyntaxParser - extension SyntaxNode { public var isUnknown: Bool { return raw.kind.isUnknown } public var asUnknown: UnknownSyntax? { @@ -1964,10 +1962,3 @@ extension Syntax { } } } - -extension SyntaxParser { - static func verifyNodeDeclarationHash() -> Bool { - return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == - "e9565bceebb81b9c3a69c442a8576b029d7eaf9c" - } -} diff --git a/Sources/SwiftSyntaxParser/CNodes.swift b/Sources/SwiftSyntaxParser/CNodes.swift new file mode 100644 index 00000000000..15f69ccf91a --- /dev/null +++ b/Sources/SwiftSyntaxParser/CNodes.swift @@ -0,0 +1,73 @@ +//===---------------------------- CNodes.swift ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +typealias CSyntaxNode = swiftparse_syntax_node_t +typealias CTriviaPiece = swiftparse_trivia_piece_t +typealias CSyntaxNodePtr = UnsafePointer +typealias CTriviaPiecePtr = UnsafePointer +typealias CSyntaxKind = swiftparse_syntax_kind_t +typealias CTokenKind = swiftparse_token_kind_t +typealias CTriviaKind = swiftparse_trivia_kind_t +typealias CTokenData = swiftparse_token_data_t +typealias CLayoutData = swiftparse_layout_data_t +typealias CParseLookupResult = swiftparse_lookup_result_t +typealias CClientNode = swiftparse_client_node_t +typealias CDiagnostic = swiftparser_diagnostic_t +typealias CFixit = swiftparse_diagnostic_fixit_t +typealias CRange = swiftparse_range_t + +/// Computes a hash value that describes the layout of all C nodes which are +/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`. +/// This should match the value returned by the `cNodeLayoutHash` function in +/// the `SwiftSyntax` module. +public func cNodeLayoutHash() -> Int { + var hasher = Hasher() + + // These two types are not defined in terms of the C types in SwiftSyntax. + // Let's include them specifically in the hash as well. + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.offset)) + hasher.combine(MemoryLayout.offset(of: \.length)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.length)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia)) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.range)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.nodes)) + hasher.combine(MemoryLayout.offset(of: \.nodes_count)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.present)) + + return hasher.finalize() +} diff --git a/Sources/SwiftSyntax/Diagnostic.swift b/Sources/SwiftSyntaxParser/Diagnostic.swift similarity index 99% rename from Sources/SwiftSyntax/Diagnostic.swift rename to Sources/SwiftSyntaxParser/Diagnostic.swift index e09cdaf8292..f267bc672fb 100644 --- a/Sources/SwiftSyntax/Diagnostic.swift +++ b/Sources/SwiftSyntaxParser/Diagnostic.swift @@ -12,6 +12,8 @@ // This file provides the Diagnostic, Note, and FixIt types. //===----------------------------------------------------------------------===// +import SwiftSyntax + /// A FixIt represents a change to source code in order to "correct" a /// diagnostic. public enum FixIt: Codable, CustomDebugStringConvertible { diff --git a/Sources/SwiftSyntax/DiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/DiagnosticConsumer.swift similarity index 100% rename from Sources/SwiftSyntax/DiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/DiagnosticConsumer.swift diff --git a/Sources/SwiftSyntax/DiagnosticEngine.swift b/Sources/SwiftSyntaxParser/DiagnosticEngine.swift similarity index 99% rename from Sources/SwiftSyntax/DiagnosticEngine.swift rename to Sources/SwiftSyntaxParser/DiagnosticEngine.swift index e3727811ec8..51faf7197bd 100644 --- a/Sources/SwiftSyntax/DiagnosticEngine.swift +++ b/Sources/SwiftSyntaxParser/DiagnosticEngine.swift @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// import Foundation +import SwiftSyntax /// The DiagnosticEngine allows Swift tools to emit diagnostics. public class DiagnosticEngine { diff --git a/Sources/SwiftSyntax/JSONDiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift similarity index 99% rename from Sources/SwiftSyntax/JSONDiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift index 2adf80527fd..caa7cec03d6 100644 --- a/Sources/SwiftSyntax/JSONDiagnosticConsumer.swift +++ b/Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// import Foundation +import SwiftSyntax public final class JSONDiagnosticConsumer: DiagnosticConsumer { /// Enumerates the possible places this consumer might output diagnostics. diff --git a/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb b/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb new file mode 100644 index 00000000000..7ccc18db392 --- /dev/null +++ b/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb @@ -0,0 +1,27 @@ +%{ + # -*- mode: Swift -*- + from gyb_syntax_support import calculate_node_hash + # Ignore the following admonition it applies to the resulting .swift file only +}% +//// Automatically Generated From NodeDeclarationHash.swift.gyb. +//// Do Not Edit Directly! +//===--------------------- NodeDeclarationHash.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +extension SyntaxParser { + static func verifyNodeDeclarationHash() -> Bool { + return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == + "${calculate_node_hash()}" + } +} diff --git a/Sources/SwiftSyntax/PrintingDiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/PrintingDiagnosticConsumer.swift similarity index 100% rename from Sources/SwiftSyntax/PrintingDiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/PrintingDiagnosticConsumer.swift diff --git a/Sources/SwiftSyntax/SyntaxParser.swift b/Sources/SwiftSyntaxParser/SyntaxParser.swift similarity index 86% rename from Sources/SwiftSyntax/SyntaxParser.swift rename to Sources/SwiftSyntaxParser/SyntaxParser.swift index 4653e5f4d95..71023ae2d68 100644 --- a/Sources/SwiftSyntax/SyntaxParser.swift +++ b/Sources/SwiftSyntaxParser/SyntaxParser.swift @@ -16,21 +16,7 @@ @_implementationOnly import _InternalSwiftSyntaxParser import Foundation - -typealias CSyntaxNode = swiftparse_syntax_node_t -typealias CTriviaPiece = swiftparse_trivia_piece_t -typealias CSyntaxNodePtr = UnsafePointer -typealias CTriviaPiecePtr = UnsafePointer -typealias CSyntaxKind = swiftparse_syntax_kind_t -typealias CTokenKind = swiftparse_token_kind_t -typealias CTriviaKind = swiftparse_trivia_kind_t -typealias CTokenData = swiftparse_token_data_t -typealias CLayoutData = swiftparse_layout_data_t -typealias CParseLookupResult = swiftparse_lookup_result_t -typealias CClientNode = swiftparse_client_node_t -typealias CDiagnostic = swiftparser_diagnostic_t -typealias CFixit = swiftparse_diagnostic_fixit_t -typealias CRange = swiftparse_range_t +import SwiftSyntax /// A list of possible errors that could be encountered while parsing a /// Syntax tree. @@ -64,6 +50,7 @@ public enum SyntaxParser { /// Incompatibility can occur if the loaded `lib_InternalSwiftSyntaxParser.dylib/.so` /// is from a toolchain that is not compatible with this version of SwiftSyntax. fileprivate static var nodeHashVerifyResult: Bool = verifyNodeDeclarationHash() + fileprivate static var cnodeLayoutHashVerifyResult: Bool = verifyCNodeLayoutHash() /// Parses the string into a full-fidelity Syntax tree. /// @@ -82,7 +69,7 @@ public enum SyntaxParser { filenameForDiagnostics: String = "", diagnosticEngine: DiagnosticEngine? = nil ) throws -> SourceFileSyntax { - guard nodeHashVerifyResult else { + guard nodeHashVerifyResult && cnodeLayoutHashVerifyResult else { throw ParserError.parserCompatibilityCheckFailed } // Get a native UTF8 string for efficient indexing with UTF8 byte offsets. @@ -94,7 +81,7 @@ public enum SyntaxParser { let rawSyntax = parseRaw(utf8Source, parseTransition, filenameForDiagnostics, diagnosticEngine) - let base = Syntax(SyntaxData.forRoot(rawSyntax)) + let base = _SyntaxParserInterop.nodeFromRetainedOpaqueRawSyntax(rawSyntax) guard let file = base.as(SourceFileSyntax.self) else { throw ParserError.invalidSyntaxData } @@ -115,7 +102,7 @@ public enum SyntaxParser { // Avoid using `String(contentsOf:)` because it creates a wrapped NSString. let fileData = try Data(contentsOf: url) let source = fileData.withUnsafeBytes { buf in - return String.fromBuffer(buf.bindMemory(to: UInt8.self)) + return String(decoding: buf.bindMemory(to: UInt8.self), as: UTF8.self) } return try parse(source: source, filenameForDiagnostics: url.path, diagnosticEngine: diagnosticEngine) @@ -126,7 +113,7 @@ public enum SyntaxParser { _ parseTransition: IncrementalParseTransition?, _ filenameForDiagnostics: String, _ diagnosticEngine: DiagnosticEngine? - ) -> RawSyntax { + ) -> CClientNode { precondition(source.isContiguousUTF8) let c_parser = swiftparse_parser_create() defer { @@ -134,11 +121,7 @@ public enum SyntaxParser { } let nodeHandler = { (cnode: CSyntaxNodePtr!) -> UnsafeMutableRawPointer in - let node = RawSyntax.create(from: cnode, source: source) - // Transfer ownership of the object to the C parser. We get ownership back - // via `moveFromCRawNode()`. - let bits = Unmanaged.passRetained(node) - return bits.toOpaque() + return _SyntaxParserInterop.getRetainedOpaqueRawSyntax(cnode: cnode, source: source) } swiftparse_parser_set_node_handler(c_parser, nodeHandler); @@ -147,12 +130,12 @@ public enum SyntaxParser { let nodeLookup = { (offset: Int, kind: CSyntaxKind) -> CParseLookupResult in guard let foundNode = - parseLookup.lookUp(offset, kind: .fromRawValue(kind)) else { + parseLookup.lookUp(offset, kind: kind) else { return CParseLookupResult(length: 0, node: nil) } let lengthToSkip = foundNode.byteSize - let bits = Unmanaged.passRetained(foundNode.raw) - return CParseLookupResult(length: lengthToSkip, node: bits.toOpaque()) + let opaqueNode = _SyntaxParserInterop.getRetainedOpaqueRawSyntax(node: foundNode) + return CParseLookupResult(length: lengthToSkip, node: opaqueNode) } swiftparse_parser_set_node_lookup(c_parser, nodeLookup); } @@ -196,8 +179,7 @@ public enum SyntaxParser { swiftparse_parse_string(c_parser, buf, source.utf8.count) } - // Get ownership back from the C parser. - return RawSyntax.moveFromOpaque(c_top)! + return c_top! } } @@ -272,3 +254,7 @@ extension Diagnostic { highlights: diag.highlights, fixIts: diag.fixIts) } } + +fileprivate func verifyCNodeLayoutHash() -> Bool { + return cNodeLayoutHash() == SwiftSyntax.cNodeLayoutHash() +} diff --git a/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift b/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift new file mode 100644 index 00000000000..71170f82476 --- /dev/null +++ b/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift @@ -0,0 +1,22 @@ +//// Automatically Generated From NodeDeclarationHash.swift.gyb. +//// Do Not Edit Directly! +//===--------------------- NodeDeclarationHash.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +extension SyntaxParser { + static func verifyNodeDeclarationHash() -> Bool { + return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == + "e9565bceebb81b9c3a69c442a8576b029d7eaf9c" + } +} diff --git a/Sources/_CSwiftSyntax/include/c-syntax-nodes.h b/Sources/_CSwiftSyntax/include/c-syntax-nodes.h new file mode 100644 index 00000000000..bb64edb9cb5 --- /dev/null +++ b/Sources/_CSwiftSyntax/include/c-syntax-nodes.h @@ -0,0 +1,76 @@ +//===------------------ SwiftSyntaxCDataTypes.h -------------------*- C -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// There are two copies of this file in the Swift compiler repository and in // +// the SwiftSyntax repository. They need to always share the same contents. // +// // +// The reason behind this is that the SwiftSyntax module should be able to // +// build with no dependencies to the compiler repo (in particular not // +// _InternalSwiftSyntaxParser) so that it can be used for code generation // +// without a matching toolchain. But to make SwiftSyntax parsing efficient, // +// we need to create RawSyntax nodes from the C nodes without any conversion // +// and thus SwiftSyntax needs to have knowledge about the layout of these C // +// types. Thus the two copies of the file. The equality of these files is // +// checked in CI (verify the source matches) and at runtime by // +// SwiftSyntaxParser (verify that a hash generated from the layouts matches) // +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_C_SYNTAX_C_DATA_TYPES_H +#define SWIFT_C_SYNTAX_C_DATA_TYPES_H + +#include + +/// Offset+length in UTF8 bytes. +typedef struct { + uint32_t offset; + uint32_t length; +} swiftparse_range_t; + +typedef uint8_t swiftparse_trivia_kind_t; +typedef uint8_t swiftparse_token_kind_t; +typedef uint16_t swiftparse_syntax_kind_t; + +/// This is for the client to provide an opaque pointer that the parser will +/// associate with a syntax node. +typedef void *swiftparse_client_node_t; + +typedef struct { + /// The length in source this trivia piece occupies, in UTF8 bytes. + uint32_t length; + swiftparse_trivia_kind_t kind; +} swiftparse_trivia_piece_t; + +typedef struct { + const swiftparse_trivia_piece_t *leading_trivia; + const swiftparse_trivia_piece_t *trailing_trivia; + uint16_t leading_trivia_count; + uint16_t trailing_trivia_count; + swiftparse_token_kind_t kind; + /// Represents the range for the node, including trivia. + swiftparse_range_t range; +} swiftparse_token_data_t; + +typedef struct { + const swiftparse_client_node_t *nodes; + uint32_t nodes_count; +} swiftparse_layout_data_t; + +typedef struct { + union { + swiftparse_token_data_t token_data; + swiftparse_layout_data_t layout_data; + }; + /// The syntax kind. A value of '0' means this is a token node. + swiftparse_syntax_kind_t kind; + bool present; +} swiftparse_syntax_node_t; + +#endif diff --git a/Sources/lit-test-helper/main.swift b/Sources/lit-test-helper/main.swift index 7684edee6b3..692a4e934f9 100644 --- a/Sources/lit-test-helper/main.swift +++ b/Sources/lit-test-helper/main.swift @@ -12,6 +12,7 @@ import SwiftSyntax +import SwiftSyntaxParser import Foundation /// Print the given message to stderr diff --git a/Tests/PerformanceTest/ParsingPerformanceTests.swift b/Tests/PerformanceTest/ParsingPerformanceTests.swift index efe8938ec41..f4685a355e1 100644 --- a/Tests/PerformanceTest/ParsingPerformanceTests.swift +++ b/Tests/PerformanceTest/ParsingPerformanceTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class ParsingPerformanceTests: XCTestCase { diff --git a/Tests/PerformanceTest/VisitorPerformanceTests.swift b/Tests/PerformanceTest/VisitorPerformanceTests.swift index 5e154390f0c..16e36003b61 100644 --- a/Tests/PerformanceTest/VisitorPerformanceTests.swift +++ b/Tests/PerformanceTest/VisitorPerformanceTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class VisitorPerformanceTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift b/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/AbsolutePositionTests.swift rename to Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift index 64d46d0f4f4..91fd757d746 100644 --- a/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift +++ b/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate class FuncRenamer: SyntaxRewriter { override func visit(_ node: FunctionDeclSyntax) -> DeclSyntax { diff --git a/Tests/SwiftSyntaxTest/ClassificationTests.swift b/Tests/SwiftSyntaxParserTest/ClassificationTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/ClassificationTests.swift rename to Tests/SwiftSyntaxParserTest/ClassificationTests.swift index 0a5dc857ed3..8638eb99d4f 100644 --- a/Tests/SwiftSyntaxTest/ClassificationTests.swift +++ b/Tests/SwiftSyntaxParserTest/ClassificationTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class ClassificationTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/DiagnosticTests.swift b/Tests/SwiftSyntaxParserTest/DiagnosticTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/DiagnosticTests.swift rename to Tests/SwiftSyntaxParserTest/DiagnosticTests.swift index c352907d2fa..15044bdfdac 100644 --- a/Tests/SwiftSyntaxTest/DiagnosticTests.swift +++ b/Tests/SwiftSyntaxParserTest/DiagnosticTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate func loc(_ file: String = #file, line: Int = #line, column: Int = #line) -> SourceLocation { diff --git a/Tests/SwiftSyntaxTest/IncrementalParsingTests.swift b/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift similarity index 98% rename from Tests/SwiftSyntaxTest/IncrementalParsingTests.swift rename to Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift index bcfb3fe8ac6..5824378d70c 100644 --- a/Tests/SwiftSyntaxTest/IncrementalParsingTests.swift +++ b/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class IncrementalParsingTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/Inputs/closure.swift b/Tests/SwiftSyntaxParserTest/Inputs/closure.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/closure.swift rename to Tests/SwiftSyntaxParserTest/Inputs/closure.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/diagnostics.swift b/Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/diagnostics.swift rename to Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/near-empty.swift b/Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/near-empty.swift rename to Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift b/Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift rename to Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/visitor.swift b/Tests/SwiftSyntaxParserTest/Inputs/visitor.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/visitor.swift rename to Tests/SwiftSyntaxParserTest/Inputs/visitor.swift diff --git a/Tests/SwiftSyntaxTest/ParseFileTests.swift b/Tests/SwiftSyntaxParserTest/ParseFileTests.swift similarity index 98% rename from Tests/SwiftSyntaxTest/ParseFileTests.swift rename to Tests/SwiftSyntaxParserTest/ParseFileTests.swift index 736a69225f5..ef1c7df4c92 100644 --- a/Tests/SwiftSyntaxTest/ParseFileTests.swift +++ b/Tests/SwiftSyntaxParserTest/ParseFileTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate struct Foo { public let x: Int diff --git a/Tests/SwiftSyntaxTest/SyntaxTests.swift b/Tests/SwiftSyntaxParserTest/SyntaxTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/SyntaxTests.swift rename to Tests/SwiftSyntaxParserTest/SyntaxTests.swift index 8656fa12ba4..b86e0e0aa5b 100644 --- a/Tests/SwiftSyntaxTest/SyntaxTests.swift +++ b/Tests/SwiftSyntaxParserTest/SyntaxTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class SyntaxTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift b/Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift rename to Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift index d2a829d4ef2..002c0b51ba3 100644 --- a/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift +++ b/Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class SyntaxVisitorTests: XCTestCase { diff --git a/Tests/SwiftSyntaxParserTest/TestUtils.swift b/Tests/SwiftSyntaxParserTest/TestUtils.swift new file mode 100644 index 00000000000..2b532be4a69 --- /dev/null +++ b/Tests/SwiftSyntaxParserTest/TestUtils.swift @@ -0,0 +1,27 @@ +import Foundation +import SwiftSyntax +import XCTest + +func getInput(_ file: String) -> URL { + var result = URL(fileURLWithPath: #file) + result.deleteLastPathComponent() + result.appendPathComponent("Inputs") + result.appendPathComponent(file) + return result +} + +extension SyntaxCollection { + /// Gets the child at the provided index in this node's present children. + /// This is not provided by the Syntax API because its performance is O(n). + /// We add it here in `SyntaxCollection` for testing purposes. + func child(at index: Int) -> Element? { + guard index >= 0 && index < self.count else { return nil } + var iter = self.makeIterator() + for _ in 0.. Element { + return child(at: index)! + } +} diff --git a/Tests/SwiftSyntaxTest/TokenTest.swift b/Tests/SwiftSyntaxParserTest/TokenTest.swift similarity index 96% rename from Tests/SwiftSyntaxTest/TokenTest.swift rename to Tests/SwiftSyntaxParserTest/TokenTest.swift index c22e8a5be9e..ee619c96de5 100644 --- a/Tests/SwiftSyntaxTest/TokenTest.swift +++ b/Tests/SwiftSyntaxParserTest/TokenTest.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class TokenTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/TestUtils.swift b/Tests/SwiftSyntaxTest/TestUtils.swift index 2b159e39d03..84b1bc8ab09 100644 --- a/Tests/SwiftSyntaxTest/TestUtils.swift +++ b/Tests/SwiftSyntaxTest/TestUtils.swift @@ -2,14 +2,6 @@ import Foundation import SwiftSyntax import XCTest -func getInput(_ file: String) -> URL { - var result = URL(fileURLWithPath: #file) - result.deleteLastPathComponent() - result.appendPathComponent("Inputs") - result.appendPathComponent(file) - return result -} - /// Verifies that there is a next item returned by the iterator and that it /// satisfies the given predicate. func XCTAssertNext( diff --git a/build-script.py b/build-script.py index 71fdbe7ac51..ff707882488 100755 --- a/build-script.py +++ b/build-script.py @@ -19,6 +19,7 @@ SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources") SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax") SWIFTSYNTAXBUILDER_DIR = os.path.join(SOURCES_DIR, "SwiftSyntaxBuilder") +SWIFTSYNTAXPARSER_DIR = os.path.join(SOURCES_DIR, "SwiftSyntaxParser") LLVM_DIR = os.path.join(WORKSPACE_DIR, "llvm-project", "llvm") SWIFT_DIR = os.path.join(WORKSPACE_DIR, "swift") @@ -166,40 +167,71 @@ def generate_single_gyb_file( check_call(rsync_command, verbose=verbose) - -def generate_gyb_files( - gyb_exec, verbose, add_source_locations, - swiftsyntax_destination=None, swiftsyntaxbuilder_destination=None +# Generate the `.swift` files for all `.gyb` files in `sources_dir`. If +# `destination_dir` is not `None`, the resulting files will be written to +# `destination_dir`, otherwise they will be written to +# `sources_dir/gyb_generated`. +def generate_gyb_files_helper( + sources_dir, + destination_dir, + gyb_exec, + add_source_locations, + verbose, ): - print("** Generating gyb Files **") - - check_gyb_exec(gyb_exec) - check_rsync() - temp_files_dir = tempfile.gettempdir() + make_dir_if_needed(temp_files_dir) - if swiftsyntax_destination is None: - swiftsyntax_destination = os.path.join( - SWIFTSYNTAX_DIR, "gyb_generated") + if destination_dir is None: + destination_dir = os.path.join(sources_dir, "gyb_generated") + make_dir_if_needed(destination_dir) - if swiftsyntaxbuilder_destination is None: - swiftsyntaxbuilder_destination = os.path.join( - SWIFTSYNTAXBUILDER_DIR, "gyb_generated") + # Clear any *.swift files that are relics from the previous run. + clear_gyb_files_from_previous_run( + sources_dir, destination_dir, verbose) + + # Generate the new .swift files in `temp_files_dir` and only copy them + # to `destiantion_dir` if they are different than the + # files already residing there. This way we don't touch the generated .swift + # files if they haven't changed and don't trigger a rebuild. + for gyb_file in os.listdir(sources_dir): + if not gyb_file.endswith(".gyb"): + continue - template_destination = os.path.join(swiftsyntax_destination, "syntax_nodes") + gyb_file_path = os.path.join(sources_dir, gyb_file) + + # Slice off the '.gyb' to get the name for the output file + output_file_name = gyb_file[:-4] + + generate_single_gyb_file( + gyb_exec, + gyb_file_path, + output_file_name, + destination_dir, + temp_files_dir, + add_source_locations, + additional_gyb_flags=[], + verbose=verbose, + ) +# Generate the syntax node `.swift` files from `SyntaxNodes.swift.gyb.template`. +# `destination_dir` is not `None`, the resulting files will be written to +# `destination_dir/syntax_nodes`, otherwise they will be written to +# `sources_dir/gyb_generated/syntax_nodes`. +def generate_syntax_node_template_gyb_files( + destination_dir, + gyb_exec, + add_source_locations, + verbose +): + temp_files_dir = tempfile.gettempdir() make_dir_if_needed(temp_files_dir) - make_dir_if_needed(swiftsyntax_destination) - make_dir_if_needed(swiftsyntaxbuilder_destination) - make_dir_if_needed(template_destination) - # Clear any *.swift files that are relics from the previous run. - clear_gyb_files_from_previous_run( - SWIFTSYNTAX_DIR, swiftsyntax_destination, verbose) + if destination_dir is None: + destination_dir = os.path.join(SWIFTSYNTAX_DIR, "gyb_generated") - clear_gyb_files_from_previous_run( - SWIFTSYNTAXBUILDER_DIR, swiftsyntaxbuilder_destination, verbose) + template_destination = os.path.join(destination_dir, "syntax_nodes") + make_dir_if_needed(template_destination) for previous_gyb_gen_file in os.listdir(template_destination): if previous_gyb_gen_file.endswith(".swift"): if previous_gyb_gen_file not in BASE_KIND_FILES.values(): @@ -209,13 +241,6 @@ def generate_gyb_files( verbose=verbose, ) - generate_gyb_files_helper(gyb_exec, SWIFTSYNTAX_DIR, swiftsyntax_destination, - temp_files_dir, add_source_locations, verbose) - - generate_gyb_files_helper(gyb_exec, SWIFTSYNTAXBUILDER_DIR, - swiftsyntaxbuilder_destination, temp_files_dir, add_source_locations, - verbose) - for base_kind in BASE_KIND_FILES: output_file_name = BASE_KIND_FILES[base_kind] @@ -234,6 +259,45 @@ def generate_gyb_files( verbose=verbose, ) + +def generate_gyb_files( + gyb_exec, verbose, add_source_locations, + swiftsyntax_destination=None, swiftsyntaxbuilder_destination=None, + swiftsyntaxparser_destination=None, +): + print("** Generating gyb Files **") + + check_gyb_exec(gyb_exec) + check_rsync() + + generate_gyb_files_helper( + SWIFTSYNTAX_DIR, + swiftsyntax_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_gyb_files_helper( + SWIFTSYNTAXBUILDER_DIR, + swiftsyntaxbuilder_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_gyb_files_helper( + SWIFTSYNTAXPARSER_DIR, + swiftsyntaxparser_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_syntax_node_template_gyb_files( + swiftsyntax_destination, + gyb_exec, + add_source_locations, + verbose + ) + print("Done Generating gyb Files") @@ -257,40 +321,6 @@ def clear_gyb_files_from_previous_run(sources_dir, destination_dir, verbose): verbose=verbose ) - -# Generate the new .swift files in `temp_files_dir` and only copy them -# to `destiantion_dir` if they are different than the -# files already residing there. This way we don't touch the generated .swift -# files if they haven't changed and don't trigger a rebuild. -def generate_gyb_files_helper( - gyb_exec, - sources_dir, - destination_dir, - temp_files_dir, - add_source_locations, - verbose -): - for gyb_file in os.listdir(sources_dir): - if not gyb_file.endswith(".gyb"): - continue - - gyb_file_path = os.path.join(sources_dir, gyb_file) - - # Slice off the '.gyb' to get the name for the output file - output_file_name = gyb_file[:-4] - - generate_single_gyb_file( - gyb_exec, - gyb_file_path, - output_file_name, - destination_dir, - temp_files_dir, - add_source_locations, - additional_gyb_flags=[], - verbose=verbose, - ) - - # ----------------------------------------------------------------------------- # Building SwiftSyntax @@ -359,13 +389,16 @@ def verify_generated_files(gyb_exec, verbose): user_swiftsyntax_generated_dir = os.path.join( SWIFTSYNTAX_DIR, "gyb_generated" ) - user_swiftsyntaxbuilder_generated_dir = os.path.join( SWIFTSYNTAXBUILDER_DIR, "gyb_generated" ) + user_swiftsyntaxparser_generated_dir = os.path.join( + SWIFTSYNTAXPARSER_DIR, "gyb_generated" + ) self_swiftsyntax_generated_dir = tempfile.mkdtemp() self_swiftsyntaxbuilder_generated_dir = tempfile.mkdtemp() + self_swiftsyntaxparser_generated_dir = tempfile.mkdtemp() generate_gyb_files( gyb_exec, @@ -373,19 +406,22 @@ def verify_generated_files(gyb_exec, verbose): add_source_locations=False, swiftsyntax_destination=self_swiftsyntax_generated_dir, swiftsyntaxbuilder_destination=self_swiftsyntaxbuilder_generated_dir, + swiftsyntaxparser_destination=self_swiftsyntaxparser_generated_dir, ) check_generated_files_match(self_swiftsyntax_generated_dir, user_swiftsyntax_generated_dir) - check_generated_files_match(self_swiftsyntaxbuilder_generated_dir, + check_generated_files_match(self_swiftsyntaxbuilder_generated_dir, user_swiftsyntaxbuilder_generated_dir) + check_generated_files_match(self_swiftsyntaxparser_generated_dir, + user_swiftsyntaxparser_generated_dir) def check_generated_files_match(self_generated_dir, user_generated_dir): command = [ "diff", - "-r", - "-x", + "--recursive", + "--exclude", ".*", # Exclude dot files like .DS_Store "--context=0", self_generated_dir, @@ -394,6 +430,22 @@ def check_generated_files_match(self_generated_dir, user_generated_dir): check_call(command) +def verify_c_syntax_nodes_match(): + print("** Validating that the C data types match **") + + swift_syntax_c_definitions = os.path.join( + SOURCES_DIR, "_CSwiftSyntax", "include", "c-syntax-nodes.h") + swiftc_c_definitions = os.path.join( + SWIFT_DIR, "include", "swift-c", "SyntaxParser", + "SwiftSyntaxCDataTypes.h") + + check_call([ + "diff", + swift_syntax_c_definitions, + swiftc_c_definitions, + ]) + + def run_tests( toolchain, build_dir, multiroot_data_file, release, filecheck_exec, skip_lit_tests, verbose @@ -790,6 +842,8 @@ def main(): if args.degyb_only: sys.exit(0) + verify_c_syntax_nodes_match() + if args.generate_xcodeproj: xcode_gen(config=args.xcconfig_path) sys.exit(0) diff --git a/utils/group.json b/utils/group.json index c324df51afb..23c40d97423 100644 --- a/utils/group.json +++ b/utils/group.json @@ -32,14 +32,6 @@ "SyntaxVisitor.swift", "Misc.swift", ], - "Diagnostic": [ - "Diagnostic.swift", - "DiagnosticConsumer.swift", - "DiagnosticEngine.swift", - "JSONDiagnosticConsumer.swift", - "PrintingDiagnosticConsumer.swift", - "SyntaxVerifier.swift", - ], "Position": [ "AbsolutePosition.swift", "SourceLength.swift", @@ -48,9 +40,11 @@ ], "Parse": [ "IncrementalParseTransition.swift", - "SyntaxParser.swift", ], "Internal": [ + "_SyntaxParserInterop.swift", + "CNodes.swift", "AtomicCounter.swift", + "SyntaxVerifier.swift", ] }