From 316f8470c05448ed90bf7caa715594fa6ae2c5f6 Mon Sep 17 00:00:00 2001 From: Alex Hoppen Date: Tue, 21 Sep 2021 17:48:12 +0200 Subject: [PATCH] Split SwiftSyntaxParser into a separate module MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When SwiftSyntax is just being used to generate code and not to parse Swift source files, there is no reason that a compatible `_InternalSwiftSyntaxParser.dylib` needs to be present. We do, however, currently have the requirement of always needing a `_InternalSwiftSyntaxParser.dylib` because `SyntaxParser` lives inside the `SwiftSyntax` module and thus `SwiftSyntax` needs to link against `_InternalSwiftSyntaxParser.dylib`. To lift the requirement, split `SwiftSyntax` into two modules: `SwiftSyntax` has no requirement on `_InternalSwiftSyntaxParser.dylib`. It only offers functionality to programatically generate `SwiftSyntax` trees and walk them. The new `SwiftSyntaxParser` module links against `_InternalSwiftSyntaxParser.dylib` and provides the ability to generate `SwiftSyntax` trees from source files. To efficiently generate `SwiftSyntax` nodes from the parser, `RawSyntax` (and its child types) does depend on the layout of the C nodes (for example, `TokenData` stores `CTriviaPiece`s as trailing trivia and only constructs the trivia’s Swift representation when requested). I don’t think it’s possible to eliminate that behaviour without eager translation and thus decided to include a copy of the C nodes inside the `SwiftSyntax`. When `SwiftSyntax` is used on its own, they are never used and thus their value is irrelevant. When a source file is parsed through `SwiftSyntaxParser`, it first verifies that the C node layouts match those defined in `_InternalSwiftSyntaxParser.dylib`, similar to how we currently compute the node hash. If the layouts are not compatible, we throw an error. We also verify that the files definingt these types match when invoking `build-script` (e.g. in CI). --- Changelog.md | 3 + Package.swift | 9 +- Sources/SwiftSyntax/CNodes.swift | 72 +++++++ .../IncrementalParseTransition.swift | 10 +- Sources/SwiftSyntax/Misc.swift.gyb | 9 - Sources/SwiftSyntax/RawSyntax.swift | 10 +- .../SwiftSyntax/_SyntaxParserInterop.swift | 54 +++++ Sources/SwiftSyntax/gyb_generated/Misc.swift | 9 - Sources/SwiftSyntaxParser/CNodes.swift | 73 +++++++ .../Diagnostic.swift | 2 + .../DiagnosticConsumer.swift | 0 .../DiagnosticEngine.swift | 1 + .../JSONDiagnosticConsumer.swift | 1 + .../NodeDeclarationHash.swift.gyb | 27 +++ .../PrintingDiagnosticConsumer.swift | 0 .../SyntaxParser.swift | 44 ++-- .../gyb_generated/NodeDeclarationHash.swift | 22 ++ .../_CSwiftSyntax/include/c-syntax-nodes.h | 76 +++++++ Sources/lit-test-helper/main.swift | 1 + .../ParsingPerformanceTests.swift | 1 + .../VisitorPerformanceTests.swift | 1 + .../AbsolutePositionTests.swift | 1 + .../ClassificationTests.swift | 1 + .../DiagnosticTests.swift | 1 + .../IncrementalParsingTests.swift | 1 + .../Inputs/closure.swift | 0 .../Inputs/diagnostics.swift | 0 .../Inputs/near-empty.swift | 0 .../Inputs/nested-blocks.swift | 0 .../Inputs/visitor.swift | 0 .../ParseFileTests.swift | 1 + .../SyntaxTests.swift | 1 + .../SyntaxVisitorTests.swift | 1 + Tests/SwiftSyntaxParserTest/TestUtils.swift | 27 +++ .../TokenTest.swift | 1 + Tests/SwiftSyntaxTest/TestUtils.swift | 8 - build-script.py | 192 +++++++++++------- utils/group.json | 12 +- 38 files changed, 526 insertions(+), 146 deletions(-) create mode 100644 Sources/SwiftSyntax/CNodes.swift create mode 100644 Sources/SwiftSyntax/_SyntaxParserInterop.swift create mode 100644 Sources/SwiftSyntaxParser/CNodes.swift rename Sources/{SwiftSyntax => SwiftSyntaxParser}/Diagnostic.swift (99%) rename Sources/{SwiftSyntax => SwiftSyntaxParser}/DiagnosticConsumer.swift (100%) rename Sources/{SwiftSyntax => SwiftSyntaxParser}/DiagnosticEngine.swift (99%) rename Sources/{SwiftSyntax => SwiftSyntaxParser}/JSONDiagnosticConsumer.swift (99%) create mode 100644 Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb rename Sources/{SwiftSyntax => SwiftSyntaxParser}/PrintingDiagnosticConsumer.swift (100%) rename Sources/{SwiftSyntax => SwiftSyntaxParser}/SyntaxParser.swift (86%) create mode 100644 Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift create mode 100644 Sources/_CSwiftSyntax/include/c-syntax-nodes.h rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/AbsolutePositionTests.swift (99%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/ClassificationTests.swift (99%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/DiagnosticTests.swift (99%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/IncrementalParsingTests.swift (98%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/Inputs/closure.swift (100%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/Inputs/diagnostics.swift (100%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/Inputs/near-empty.swift (100%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/Inputs/nested-blocks.swift (100%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/Inputs/visitor.swift (100%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/ParseFileTests.swift (98%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/SyntaxTests.swift (99%) rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/SyntaxVisitorTests.swift (99%) create mode 100644 Tests/SwiftSyntaxParserTest/TestUtils.swift rename Tests/{SwiftSyntaxTest => SwiftSyntaxParserTest}/TokenTest.swift (96%) diff --git a/Changelog.md b/Changelog.md index 49bab269cb0..9f8a6373dbd 100644 --- a/Changelog.md +++ b/Changelog.md @@ -5,6 +5,9 @@ Note: This is in reverse chronological order, so newer entries are added to the ## `main` * To clarify that the edits passed to `IncrementalParseTransition` are applied concurrently, introduce a new `ConcurrentEdit` type that provides the guarantee and allows translation of sequentially applied edits to the expected concurrent form. +* The `SwiftSyntaxParser` type and a few related types now live in their own module (also named `SwiftSyntaxParser`). This allows using `SwiftSyntax` for code generation purposes without having a compatible `_InternalSwiftSyntaxParser.dylib` around. + + `import SwiftSyntaxParser` where necessary. ## Swift 5.3 diff --git a/Package.swift b/Package.swift index 161170233a7..bb780274d0f 100644 --- a/Package.swift +++ b/Package.swift @@ -7,11 +7,13 @@ let package = Package( name: "SwiftSyntax", targets: [ .target(name: "_CSwiftSyntax"), - .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"], exclude: ["Inputs"]), + .testTarget(name: "SwiftSyntaxTest", dependencies: ["SwiftSyntax"]), .target(name: "SwiftSyntaxBuilder", dependencies: ["SwiftSyntax"]), .testTarget(name: "SwiftSyntaxBuilderTest", dependencies: ["SwiftSyntaxBuilder"]), - .target(name: "lit-test-helper", dependencies: ["SwiftSyntax"]), - .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax"]) + .target(name: "SwiftSyntaxParser", dependencies: ["SwiftSyntax"]), + .testTarget(name: "SwiftSyntaxParserTest", dependencies: ["SwiftSyntaxParser"], exclude: ["Inputs"]), + .target(name: "lit-test-helper", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]), + .testTarget(name: "PerformanceTest", dependencies: ["SwiftSyntax", "SwiftSyntaxParser"]) // Also see targets added below ] ) @@ -52,4 +54,5 @@ if ProcessInfo.processInfo.environment["SWIFT_BUILD_SCRIPT_ENVIRONMENT"] != nil } package.products.append(.library(name: "SwiftSyntax", type: libraryType, targets: ["SwiftSyntax"])) +package.products.append(.library(name: "SwiftSyntaxParser", type: libraryType, targets: ["SwiftSyntaxParser"])) package.products.append(.library(name: "SwiftSyntaxBuilder", type: libraryType, targets: ["SwiftSyntaxBuilder"])) diff --git a/Sources/SwiftSyntax/CNodes.swift b/Sources/SwiftSyntax/CNodes.swift new file mode 100644 index 00000000000..5408a46dbcc --- /dev/null +++ b/Sources/SwiftSyntax/CNodes.swift @@ -0,0 +1,72 @@ +//===---------------------------- CNodes.swift ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _CSwiftSyntax + +// These two types need to be exposed publicly, so they can't rely on the C +// types defined in _CSwiftSyntax. +public typealias CSyntaxKind = UInt16 +public typealias CClientNode = UnsafeMutableRawPointer + +typealias CSyntaxNode = swiftparse_syntax_node_t +typealias CSyntaxNodePtr = UnsafePointer +typealias CTokenKind = swiftparse_token_kind_t +typealias CTriviaKind = swiftparse_trivia_kind_t +typealias CTokenData = swiftparse_token_data_t +typealias CLayoutData = swiftparse_layout_data_t +typealias CTriviaPiecePtr = UnsafePointer +typealias CTriviaPiece = swiftparse_trivia_piece_t + +/// Computes a hash value that describes the layout of all C nodes which are +/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`. +/// This should match the value returned by the `cNodeLayoutHash` function in +/// the `SwiftSyntaxParser` module. +public func cNodeLayoutHash() -> Int { + var hasher = Hasher() + + // These two types are not defined in terms of the C types in SwiftSyntax. + // Let's include them specifically in the hash as well. + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.offset)) + hasher.combine(MemoryLayout.offset(of: \.length)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.length)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia)) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.range)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.nodes)) + hasher.combine(MemoryLayout.offset(of: \.nodes_count)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.present)) + + return hasher.finalize() +} diff --git a/Sources/SwiftSyntax/IncrementalParseTransition.swift b/Sources/SwiftSyntax/IncrementalParseTransition.swift index 0662275ecfc..624df35305e 100644 --- a/Sources/SwiftSyntax/IncrementalParseTransition.swift +++ b/Sources/SwiftSyntax/IncrementalParseTransition.swift @@ -199,11 +199,11 @@ public struct ConcurrentEdits { /// Provides a mechanism for the parser to skip regions of an incrementally /// updated source that was already parsed during a previous parse invocation. -internal struct IncrementalParseLookup { +public struct IncrementalParseLookup { fileprivate let transition: IncrementalParseTransition fileprivate var cursor: SyntaxCursor - init(transition: IncrementalParseTransition) { + public init(transition: IncrementalParseTransition) { self.transition = transition self.cursor = .init(root: transition.previousTree.data.absoluteRaw) } @@ -224,16 +224,16 @@ internal struct IncrementalParseLookup { /// /// - Parameters: /// - offset: The byte offset of the source string that is currently parsed. - /// - kind: The `SyntaxKind` that the parser expects at this position. + /// - kind: The `CSyntaxKind` that the parser expects at this position. /// - Returns: A `SyntaxNode` node from the previous parse invocation, /// representing the contents of this region, if it is still valid /// to re-use. `nil` otherwise. - mutating func lookUp(_ newOffset: Int, kind: SyntaxKind) -> SyntaxNode? { + public mutating func lookUp(_ newOffset: Int, kind: CSyntaxKind) -> SyntaxNode? { guard let prevOffset = translateToPreEditOffset(newOffset) else { return nil } let prevPosition = AbsolutePosition(utf8Offset: prevOffset) - let node = cursorLookup(prevPosition: prevPosition, kind: kind) + let node = cursorLookup(prevPosition: prevPosition, kind: .fromRawValue(kind)) if let delegate = reusedDelegate, let node = node { delegate.parserReusedNode( range: ByteSourceRange(offset: newOffset, length: node.byteSize), diff --git a/Sources/SwiftSyntax/Misc.swift.gyb b/Sources/SwiftSyntax/Misc.swift.gyb index 15be1f7c769..110f5ec8aa0 100644 --- a/Sources/SwiftSyntax/Misc.swift.gyb +++ b/Sources/SwiftSyntax/Misc.swift.gyb @@ -19,8 +19,6 @@ // //===----------------------------------------------------------------------===// -@_implementationOnly import _InternalSwiftSyntaxParser - extension SyntaxNode { public var isUnknown: Bool { return raw.kind.isUnknown } public var asUnknown: UnknownSyntax? { @@ -63,10 +61,3 @@ extension Syntax { } } } - -extension SyntaxParser { - static func verifyNodeDeclarationHash() -> Bool { - return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == - "${calculate_node_hash()}" - } -} diff --git a/Sources/SwiftSyntax/RawSyntax.swift b/Sources/SwiftSyntax/RawSyntax.swift index e6a738f444d..ff6f78d56a4 100644 --- a/Sources/SwiftSyntax/RawSyntax.swift +++ b/Sources/SwiftSyntax/RawSyntax.swift @@ -1216,12 +1216,12 @@ final class RawSyntax: ManagedBuffer { } extension RawSyntax { + static func moveFromOpaque(_ cn: CClientNode) -> RawSyntax { + return Unmanaged.fromOpaque(cn).takeRetainedValue() + } + static func moveFromOpaque(_ cn: CClientNode?) -> RawSyntax? { - if let subnode = cn { - return Unmanaged.fromOpaque(subnode).takeRetainedValue() - } else { - return nil - } + return cn.map(moveFromOpaque) } static func getFromOpaque(_ cn: CClientNode?) -> RawSyntax? { diff --git a/Sources/SwiftSyntax/_SyntaxParserInterop.swift b/Sources/SwiftSyntax/_SyntaxParserInterop.swift new file mode 100644 index 00000000000..6dd40231d21 --- /dev/null +++ b/Sources/SwiftSyntax/_SyntaxParserInterop.swift @@ -0,0 +1,54 @@ +//===--------------------- _SyntaxParserInterop.swift ---------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +/// Namespace for methods needed by the `SwiftSyntaxParser` module to +/// efficiently create `SwiftSyntax` nodes from the C nodes created by the +/// parser. +public enum _SyntaxParserInterop { + private static func getRetainedOpaque(rawSyntax: RawSyntax) -> CClientNode { + return Unmanaged.passRetained(rawSyntax).toOpaque() + } + + /// Create a `RawSyntax` node for the given `cnode` and return an opaque + /// pointer to the `RawSyntax` node (a `CClientNode`). + /// After this method finishes, the `RawSyntax` node has a retain count of 1 + /// and is owned by whoever manages the returned `CClientNode`. Passing the + /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership + /// back to `SwiftSyntax`. + public static func getRetainedOpaqueRawSyntax( + cnode: UnsafeRawPointer, source: String + ) -> CClientNode { + let cnode = cnode.assumingMemoryBound(to: CSyntaxNode.self) + // Transfer ownership of the object to the C parser. We get ownership back + // via `moveFromCRawNode()`. + let node = RawSyntax.create(from: cnode, source: source) + return getRetainedOpaque(rawSyntax: node) + } + + /// Return an opaque pointer to the given `node`. + /// After this method finishes, the `RawSyntax` node has a retain count of 1 + /// and is owned by whoever manages the returned `CClientNode`. Passing the + /// `CClientNode` to `nodeFromRetainedOpaqueRawSyntax` transfers ownership + /// back to `SwiftSyntax`. + public static func getRetainedOpaqueRawSyntax(node: SyntaxNode) + -> CClientNode { + return getRetainedOpaque(rawSyntax: node.raw) + } + + /// After an opaque pointer to a `RawSyntax` node has been created using one + /// of the methods above, transfer its ownership back to a `Syntax` node, + /// which is managed by `SwiftSyntax`. + public static func nodeFromRetainedOpaqueRawSyntax(_ cRoot: CClientNode) + -> Syntax { + return Syntax(SyntaxData.forRoot(RawSyntax.moveFromOpaque(cRoot))) + } +} diff --git a/Sources/SwiftSyntax/gyb_generated/Misc.swift b/Sources/SwiftSyntax/gyb_generated/Misc.swift index 2960af8142e..e2d633039c7 100644 --- a/Sources/SwiftSyntax/gyb_generated/Misc.swift +++ b/Sources/SwiftSyntax/gyb_generated/Misc.swift @@ -12,8 +12,6 @@ // //===----------------------------------------------------------------------===// -@_implementationOnly import _InternalSwiftSyntaxParser - extension SyntaxNode { public var isUnknown: Bool { return raw.kind.isUnknown } public var asUnknown: UnknownSyntax? { @@ -1964,10 +1962,3 @@ extension Syntax { } } } - -extension SyntaxParser { - static func verifyNodeDeclarationHash() -> Bool { - return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == - "e9565bceebb81b9c3a69c442a8576b029d7eaf9c" - } -} diff --git a/Sources/SwiftSyntaxParser/CNodes.swift b/Sources/SwiftSyntaxParser/CNodes.swift new file mode 100644 index 00000000000..15f69ccf91a --- /dev/null +++ b/Sources/SwiftSyntaxParser/CNodes.swift @@ -0,0 +1,73 @@ +//===---------------------------- CNodes.swift ----------------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +typealias CSyntaxNode = swiftparse_syntax_node_t +typealias CTriviaPiece = swiftparse_trivia_piece_t +typealias CSyntaxNodePtr = UnsafePointer +typealias CTriviaPiecePtr = UnsafePointer +typealias CSyntaxKind = swiftparse_syntax_kind_t +typealias CTokenKind = swiftparse_token_kind_t +typealias CTriviaKind = swiftparse_trivia_kind_t +typealias CTokenData = swiftparse_token_data_t +typealias CLayoutData = swiftparse_layout_data_t +typealias CParseLookupResult = swiftparse_lookup_result_t +typealias CClientNode = swiftparse_client_node_t +typealias CDiagnostic = swiftparser_diagnostic_t +typealias CFixit = swiftparse_diagnostic_fixit_t +typealias CRange = swiftparse_range_t + +/// Computes a hash value that describes the layout of all C nodes which are +/// passed as opaque values between `SwiftSyntaxParser` and `SwiftSyntax`. +/// This should match the value returned by the `cNodeLayoutHash` function in +/// the `SwiftSyntax` module. +public func cNodeLayoutHash() -> Int { + var hasher = Hasher() + + // These two types are not defined in terms of the C types in SwiftSyntax. + // Let's include them specifically in the hash as well. + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.offset)) + hasher.combine(MemoryLayout.offset(of: \.length)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.length)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia)) + hasher.combine(MemoryLayout.offset(of: \.leading_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.trailing_trivia_count)) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.range)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.nodes)) + hasher.combine(MemoryLayout.offset(of: \.nodes_count)) + + hasher.combine(MemoryLayout.size) + hasher.combine(MemoryLayout.offset(of: \.kind)) + hasher.combine(MemoryLayout.offset(of: \.present)) + + return hasher.finalize() +} diff --git a/Sources/SwiftSyntax/Diagnostic.swift b/Sources/SwiftSyntaxParser/Diagnostic.swift similarity index 99% rename from Sources/SwiftSyntax/Diagnostic.swift rename to Sources/SwiftSyntaxParser/Diagnostic.swift index e09cdaf8292..f267bc672fb 100644 --- a/Sources/SwiftSyntax/Diagnostic.swift +++ b/Sources/SwiftSyntaxParser/Diagnostic.swift @@ -12,6 +12,8 @@ // This file provides the Diagnostic, Note, and FixIt types. //===----------------------------------------------------------------------===// +import SwiftSyntax + /// A FixIt represents a change to source code in order to "correct" a /// diagnostic. public enum FixIt: Codable, CustomDebugStringConvertible { diff --git a/Sources/SwiftSyntax/DiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/DiagnosticConsumer.swift similarity index 100% rename from Sources/SwiftSyntax/DiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/DiagnosticConsumer.swift diff --git a/Sources/SwiftSyntax/DiagnosticEngine.swift b/Sources/SwiftSyntaxParser/DiagnosticEngine.swift similarity index 99% rename from Sources/SwiftSyntax/DiagnosticEngine.swift rename to Sources/SwiftSyntaxParser/DiagnosticEngine.swift index e3727811ec8..51faf7197bd 100644 --- a/Sources/SwiftSyntax/DiagnosticEngine.swift +++ b/Sources/SwiftSyntaxParser/DiagnosticEngine.swift @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// import Foundation +import SwiftSyntax /// The DiagnosticEngine allows Swift tools to emit diagnostics. public class DiagnosticEngine { diff --git a/Sources/SwiftSyntax/JSONDiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift similarity index 99% rename from Sources/SwiftSyntax/JSONDiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift index 2adf80527fd..caa7cec03d6 100644 --- a/Sources/SwiftSyntax/JSONDiagnosticConsumer.swift +++ b/Sources/SwiftSyntaxParser/JSONDiagnosticConsumer.swift @@ -14,6 +14,7 @@ //===----------------------------------------------------------------------===// import Foundation +import SwiftSyntax public final class JSONDiagnosticConsumer: DiagnosticConsumer { /// Enumerates the possible places this consumer might output diagnostics. diff --git a/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb b/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb new file mode 100644 index 00000000000..7ccc18db392 --- /dev/null +++ b/Sources/SwiftSyntaxParser/NodeDeclarationHash.swift.gyb @@ -0,0 +1,27 @@ +%{ + # -*- mode: Swift -*- + from gyb_syntax_support import calculate_node_hash + # Ignore the following admonition it applies to the resulting .swift file only +}% +//// Automatically Generated From NodeDeclarationHash.swift.gyb. +//// Do Not Edit Directly! +//===--------------------- NodeDeclarationHash.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +extension SyntaxParser { + static func verifyNodeDeclarationHash() -> Bool { + return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == + "${calculate_node_hash()}" + } +} diff --git a/Sources/SwiftSyntax/PrintingDiagnosticConsumer.swift b/Sources/SwiftSyntaxParser/PrintingDiagnosticConsumer.swift similarity index 100% rename from Sources/SwiftSyntax/PrintingDiagnosticConsumer.swift rename to Sources/SwiftSyntaxParser/PrintingDiagnosticConsumer.swift diff --git a/Sources/SwiftSyntax/SyntaxParser.swift b/Sources/SwiftSyntaxParser/SyntaxParser.swift similarity index 86% rename from Sources/SwiftSyntax/SyntaxParser.swift rename to Sources/SwiftSyntaxParser/SyntaxParser.swift index 4653e5f4d95..71023ae2d68 100644 --- a/Sources/SwiftSyntax/SyntaxParser.swift +++ b/Sources/SwiftSyntaxParser/SyntaxParser.swift @@ -16,21 +16,7 @@ @_implementationOnly import _InternalSwiftSyntaxParser import Foundation - -typealias CSyntaxNode = swiftparse_syntax_node_t -typealias CTriviaPiece = swiftparse_trivia_piece_t -typealias CSyntaxNodePtr = UnsafePointer -typealias CTriviaPiecePtr = UnsafePointer -typealias CSyntaxKind = swiftparse_syntax_kind_t -typealias CTokenKind = swiftparse_token_kind_t -typealias CTriviaKind = swiftparse_trivia_kind_t -typealias CTokenData = swiftparse_token_data_t -typealias CLayoutData = swiftparse_layout_data_t -typealias CParseLookupResult = swiftparse_lookup_result_t -typealias CClientNode = swiftparse_client_node_t -typealias CDiagnostic = swiftparser_diagnostic_t -typealias CFixit = swiftparse_diagnostic_fixit_t -typealias CRange = swiftparse_range_t +import SwiftSyntax /// A list of possible errors that could be encountered while parsing a /// Syntax tree. @@ -64,6 +50,7 @@ public enum SyntaxParser { /// Incompatibility can occur if the loaded `lib_InternalSwiftSyntaxParser.dylib/.so` /// is from a toolchain that is not compatible with this version of SwiftSyntax. fileprivate static var nodeHashVerifyResult: Bool = verifyNodeDeclarationHash() + fileprivate static var cnodeLayoutHashVerifyResult: Bool = verifyCNodeLayoutHash() /// Parses the string into a full-fidelity Syntax tree. /// @@ -82,7 +69,7 @@ public enum SyntaxParser { filenameForDiagnostics: String = "", diagnosticEngine: DiagnosticEngine? = nil ) throws -> SourceFileSyntax { - guard nodeHashVerifyResult else { + guard nodeHashVerifyResult && cnodeLayoutHashVerifyResult else { throw ParserError.parserCompatibilityCheckFailed } // Get a native UTF8 string for efficient indexing with UTF8 byte offsets. @@ -94,7 +81,7 @@ public enum SyntaxParser { let rawSyntax = parseRaw(utf8Source, parseTransition, filenameForDiagnostics, diagnosticEngine) - let base = Syntax(SyntaxData.forRoot(rawSyntax)) + let base = _SyntaxParserInterop.nodeFromRetainedOpaqueRawSyntax(rawSyntax) guard let file = base.as(SourceFileSyntax.self) else { throw ParserError.invalidSyntaxData } @@ -115,7 +102,7 @@ public enum SyntaxParser { // Avoid using `String(contentsOf:)` because it creates a wrapped NSString. let fileData = try Data(contentsOf: url) let source = fileData.withUnsafeBytes { buf in - return String.fromBuffer(buf.bindMemory(to: UInt8.self)) + return String(decoding: buf.bindMemory(to: UInt8.self), as: UTF8.self) } return try parse(source: source, filenameForDiagnostics: url.path, diagnosticEngine: diagnosticEngine) @@ -126,7 +113,7 @@ public enum SyntaxParser { _ parseTransition: IncrementalParseTransition?, _ filenameForDiagnostics: String, _ diagnosticEngine: DiagnosticEngine? - ) -> RawSyntax { + ) -> CClientNode { precondition(source.isContiguousUTF8) let c_parser = swiftparse_parser_create() defer { @@ -134,11 +121,7 @@ public enum SyntaxParser { } let nodeHandler = { (cnode: CSyntaxNodePtr!) -> UnsafeMutableRawPointer in - let node = RawSyntax.create(from: cnode, source: source) - // Transfer ownership of the object to the C parser. We get ownership back - // via `moveFromCRawNode()`. - let bits = Unmanaged.passRetained(node) - return bits.toOpaque() + return _SyntaxParserInterop.getRetainedOpaqueRawSyntax(cnode: cnode, source: source) } swiftparse_parser_set_node_handler(c_parser, nodeHandler); @@ -147,12 +130,12 @@ public enum SyntaxParser { let nodeLookup = { (offset: Int, kind: CSyntaxKind) -> CParseLookupResult in guard let foundNode = - parseLookup.lookUp(offset, kind: .fromRawValue(kind)) else { + parseLookup.lookUp(offset, kind: kind) else { return CParseLookupResult(length: 0, node: nil) } let lengthToSkip = foundNode.byteSize - let bits = Unmanaged.passRetained(foundNode.raw) - return CParseLookupResult(length: lengthToSkip, node: bits.toOpaque()) + let opaqueNode = _SyntaxParserInterop.getRetainedOpaqueRawSyntax(node: foundNode) + return CParseLookupResult(length: lengthToSkip, node: opaqueNode) } swiftparse_parser_set_node_lookup(c_parser, nodeLookup); } @@ -196,8 +179,7 @@ public enum SyntaxParser { swiftparse_parse_string(c_parser, buf, source.utf8.count) } - // Get ownership back from the C parser. - return RawSyntax.moveFromOpaque(c_top)! + return c_top! } } @@ -272,3 +254,7 @@ extension Diagnostic { highlights: diag.highlights, fixIts: diag.fixIts) } } + +fileprivate func verifyCNodeLayoutHash() -> Bool { + return cNodeLayoutHash() == SwiftSyntax.cNodeLayoutHash() +} diff --git a/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift b/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift new file mode 100644 index 00000000000..71170f82476 --- /dev/null +++ b/Sources/SwiftSyntaxParser/gyb_generated/NodeDeclarationHash.swift @@ -0,0 +1,22 @@ +//// Automatically Generated From NodeDeclarationHash.swift.gyb. +//// Do Not Edit Directly! +//===--------------------- NodeDeclarationHash.swift ----------------------===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// + +@_implementationOnly import _InternalSwiftSyntaxParser + +extension SyntaxParser { + static func verifyNodeDeclarationHash() -> Bool { + return String(cString: swiftparse_syntax_structure_versioning_identifier()!) == + "e9565bceebb81b9c3a69c442a8576b029d7eaf9c" + } +} diff --git a/Sources/_CSwiftSyntax/include/c-syntax-nodes.h b/Sources/_CSwiftSyntax/include/c-syntax-nodes.h new file mode 100644 index 00000000000..bb64edb9cb5 --- /dev/null +++ b/Sources/_CSwiftSyntax/include/c-syntax-nodes.h @@ -0,0 +1,76 @@ +//===------------------ SwiftSyntaxCDataTypes.h -------------------*- C -*-===// +// +// This source file is part of the Swift.org open source project +// +// Copyright (c) 2014 - 2021 Apple Inc. and the Swift project authors +// Licensed under Apache License v2.0 with Runtime Library Exception +// +// See https://swift.org/LICENSE.txt for license information +// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors +// +//===----------------------------------------------------------------------===// +// There are two copies of this file in the Swift compiler repository and in // +// the SwiftSyntax repository. They need to always share the same contents. // +// // +// The reason behind this is that the SwiftSyntax module should be able to // +// build with no dependencies to the compiler repo (in particular not // +// _InternalSwiftSyntaxParser) so that it can be used for code generation // +// without a matching toolchain. But to make SwiftSyntax parsing efficient, // +// we need to create RawSyntax nodes from the C nodes without any conversion // +// and thus SwiftSyntax needs to have knowledge about the layout of these C // +// types. Thus the two copies of the file. The equality of these files is // +// checked in CI (verify the source matches) and at runtime by // +// SwiftSyntaxParser (verify that a hash generated from the layouts matches) // +//===----------------------------------------------------------------------===// + +#ifndef SWIFT_C_SYNTAX_C_DATA_TYPES_H +#define SWIFT_C_SYNTAX_C_DATA_TYPES_H + +#include + +/// Offset+length in UTF8 bytes. +typedef struct { + uint32_t offset; + uint32_t length; +} swiftparse_range_t; + +typedef uint8_t swiftparse_trivia_kind_t; +typedef uint8_t swiftparse_token_kind_t; +typedef uint16_t swiftparse_syntax_kind_t; + +/// This is for the client to provide an opaque pointer that the parser will +/// associate with a syntax node. +typedef void *swiftparse_client_node_t; + +typedef struct { + /// The length in source this trivia piece occupies, in UTF8 bytes. + uint32_t length; + swiftparse_trivia_kind_t kind; +} swiftparse_trivia_piece_t; + +typedef struct { + const swiftparse_trivia_piece_t *leading_trivia; + const swiftparse_trivia_piece_t *trailing_trivia; + uint16_t leading_trivia_count; + uint16_t trailing_trivia_count; + swiftparse_token_kind_t kind; + /// Represents the range for the node, including trivia. + swiftparse_range_t range; +} swiftparse_token_data_t; + +typedef struct { + const swiftparse_client_node_t *nodes; + uint32_t nodes_count; +} swiftparse_layout_data_t; + +typedef struct { + union { + swiftparse_token_data_t token_data; + swiftparse_layout_data_t layout_data; + }; + /// The syntax kind. A value of '0' means this is a token node. + swiftparse_syntax_kind_t kind; + bool present; +} swiftparse_syntax_node_t; + +#endif diff --git a/Sources/lit-test-helper/main.swift b/Sources/lit-test-helper/main.swift index 7684edee6b3..692a4e934f9 100644 --- a/Sources/lit-test-helper/main.swift +++ b/Sources/lit-test-helper/main.swift @@ -12,6 +12,7 @@ import SwiftSyntax +import SwiftSyntaxParser import Foundation /// Print the given message to stderr diff --git a/Tests/PerformanceTest/ParsingPerformanceTests.swift b/Tests/PerformanceTest/ParsingPerformanceTests.swift index efe8938ec41..f4685a355e1 100644 --- a/Tests/PerformanceTest/ParsingPerformanceTests.swift +++ b/Tests/PerformanceTest/ParsingPerformanceTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class ParsingPerformanceTests: XCTestCase { diff --git a/Tests/PerformanceTest/VisitorPerformanceTests.swift b/Tests/PerformanceTest/VisitorPerformanceTests.swift index 5e154390f0c..16e36003b61 100644 --- a/Tests/PerformanceTest/VisitorPerformanceTests.swift +++ b/Tests/PerformanceTest/VisitorPerformanceTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class VisitorPerformanceTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift b/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/AbsolutePositionTests.swift rename to Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift index 64d46d0f4f4..91fd757d746 100644 --- a/Tests/SwiftSyntaxTest/AbsolutePositionTests.swift +++ b/Tests/SwiftSyntaxParserTest/AbsolutePositionTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate class FuncRenamer: SyntaxRewriter { override func visit(_ node: FunctionDeclSyntax) -> DeclSyntax { diff --git a/Tests/SwiftSyntaxTest/ClassificationTests.swift b/Tests/SwiftSyntaxParserTest/ClassificationTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/ClassificationTests.swift rename to Tests/SwiftSyntaxParserTest/ClassificationTests.swift index 0a5dc857ed3..8638eb99d4f 100644 --- a/Tests/SwiftSyntaxTest/ClassificationTests.swift +++ b/Tests/SwiftSyntaxParserTest/ClassificationTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class ClassificationTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/DiagnosticTests.swift b/Tests/SwiftSyntaxParserTest/DiagnosticTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/DiagnosticTests.swift rename to Tests/SwiftSyntaxParserTest/DiagnosticTests.swift index c352907d2fa..15044bdfdac 100644 --- a/Tests/SwiftSyntaxTest/DiagnosticTests.swift +++ b/Tests/SwiftSyntaxParserTest/DiagnosticTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate func loc(_ file: String = #file, line: Int = #line, column: Int = #line) -> SourceLocation { diff --git a/Tests/SwiftSyntaxTest/IncrementalParsingTests.swift b/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift similarity index 98% rename from Tests/SwiftSyntaxTest/IncrementalParsingTests.swift rename to Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift index bcfb3fe8ac6..5824378d70c 100644 --- a/Tests/SwiftSyntaxTest/IncrementalParsingTests.swift +++ b/Tests/SwiftSyntaxParserTest/IncrementalParsingTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class IncrementalParsingTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/Inputs/closure.swift b/Tests/SwiftSyntaxParserTest/Inputs/closure.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/closure.swift rename to Tests/SwiftSyntaxParserTest/Inputs/closure.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/diagnostics.swift b/Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/diagnostics.swift rename to Tests/SwiftSyntaxParserTest/Inputs/diagnostics.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/near-empty.swift b/Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/near-empty.swift rename to Tests/SwiftSyntaxParserTest/Inputs/near-empty.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift b/Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/nested-blocks.swift rename to Tests/SwiftSyntaxParserTest/Inputs/nested-blocks.swift diff --git a/Tests/SwiftSyntaxTest/Inputs/visitor.swift b/Tests/SwiftSyntaxParserTest/Inputs/visitor.swift similarity index 100% rename from Tests/SwiftSyntaxTest/Inputs/visitor.swift rename to Tests/SwiftSyntaxParserTest/Inputs/visitor.swift diff --git a/Tests/SwiftSyntaxTest/ParseFileTests.swift b/Tests/SwiftSyntaxParserTest/ParseFileTests.swift similarity index 98% rename from Tests/SwiftSyntaxTest/ParseFileTests.swift rename to Tests/SwiftSyntaxParserTest/ParseFileTests.swift index 736a69225f5..ef1c7df4c92 100644 --- a/Tests/SwiftSyntaxTest/ParseFileTests.swift +++ b/Tests/SwiftSyntaxParserTest/ParseFileTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser fileprivate struct Foo { public let x: Int diff --git a/Tests/SwiftSyntaxTest/SyntaxTests.swift b/Tests/SwiftSyntaxParserTest/SyntaxTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/SyntaxTests.swift rename to Tests/SwiftSyntaxParserTest/SyntaxTests.swift index 8656fa12ba4..b86e0e0aa5b 100644 --- a/Tests/SwiftSyntaxTest/SyntaxTests.swift +++ b/Tests/SwiftSyntaxParserTest/SyntaxTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class SyntaxTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift b/Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift similarity index 99% rename from Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift rename to Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift index d2a829d4ef2..002c0b51ba3 100644 --- a/Tests/SwiftSyntaxTest/SyntaxVisitorTests.swift +++ b/Tests/SwiftSyntaxParserTest/SyntaxVisitorTests.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class SyntaxVisitorTests: XCTestCase { diff --git a/Tests/SwiftSyntaxParserTest/TestUtils.swift b/Tests/SwiftSyntaxParserTest/TestUtils.swift new file mode 100644 index 00000000000..2b532be4a69 --- /dev/null +++ b/Tests/SwiftSyntaxParserTest/TestUtils.swift @@ -0,0 +1,27 @@ +import Foundation +import SwiftSyntax +import XCTest + +func getInput(_ file: String) -> URL { + var result = URL(fileURLWithPath: #file) + result.deleteLastPathComponent() + result.appendPathComponent("Inputs") + result.appendPathComponent(file) + return result +} + +extension SyntaxCollection { + /// Gets the child at the provided index in this node's present children. + /// This is not provided by the Syntax API because its performance is O(n). + /// We add it here in `SyntaxCollection` for testing purposes. + func child(at index: Int) -> Element? { + guard index >= 0 && index < self.count else { return nil } + var iter = self.makeIterator() + for _ in 0.. Element { + return child(at: index)! + } +} diff --git a/Tests/SwiftSyntaxTest/TokenTest.swift b/Tests/SwiftSyntaxParserTest/TokenTest.swift similarity index 96% rename from Tests/SwiftSyntaxTest/TokenTest.swift rename to Tests/SwiftSyntaxParserTest/TokenTest.swift index c22e8a5be9e..ee619c96de5 100644 --- a/Tests/SwiftSyntaxTest/TokenTest.swift +++ b/Tests/SwiftSyntaxParserTest/TokenTest.swift @@ -1,5 +1,6 @@ import XCTest import SwiftSyntax +import SwiftSyntaxParser public class TokenTests: XCTestCase { diff --git a/Tests/SwiftSyntaxTest/TestUtils.swift b/Tests/SwiftSyntaxTest/TestUtils.swift index 2b159e39d03..84b1bc8ab09 100644 --- a/Tests/SwiftSyntaxTest/TestUtils.swift +++ b/Tests/SwiftSyntaxTest/TestUtils.swift @@ -2,14 +2,6 @@ import Foundation import SwiftSyntax import XCTest -func getInput(_ file: String) -> URL { - var result = URL(fileURLWithPath: #file) - result.deleteLastPathComponent() - result.appendPathComponent("Inputs") - result.appendPathComponent(file) - return result -} - /// Verifies that there is a next item returned by the iterator and that it /// satisfies the given predicate. func XCTAssertNext( diff --git a/build-script.py b/build-script.py index 71fdbe7ac51..ff707882488 100755 --- a/build-script.py +++ b/build-script.py @@ -19,6 +19,7 @@ SOURCES_DIR = os.path.join(PACKAGE_DIR, "Sources") SWIFTSYNTAX_DIR = os.path.join(SOURCES_DIR, "SwiftSyntax") SWIFTSYNTAXBUILDER_DIR = os.path.join(SOURCES_DIR, "SwiftSyntaxBuilder") +SWIFTSYNTAXPARSER_DIR = os.path.join(SOURCES_DIR, "SwiftSyntaxParser") LLVM_DIR = os.path.join(WORKSPACE_DIR, "llvm-project", "llvm") SWIFT_DIR = os.path.join(WORKSPACE_DIR, "swift") @@ -166,40 +167,71 @@ def generate_single_gyb_file( check_call(rsync_command, verbose=verbose) - -def generate_gyb_files( - gyb_exec, verbose, add_source_locations, - swiftsyntax_destination=None, swiftsyntaxbuilder_destination=None +# Generate the `.swift` files for all `.gyb` files in `sources_dir`. If +# `destination_dir` is not `None`, the resulting files will be written to +# `destination_dir`, otherwise they will be written to +# `sources_dir/gyb_generated`. +def generate_gyb_files_helper( + sources_dir, + destination_dir, + gyb_exec, + add_source_locations, + verbose, ): - print("** Generating gyb Files **") - - check_gyb_exec(gyb_exec) - check_rsync() - temp_files_dir = tempfile.gettempdir() + make_dir_if_needed(temp_files_dir) - if swiftsyntax_destination is None: - swiftsyntax_destination = os.path.join( - SWIFTSYNTAX_DIR, "gyb_generated") + if destination_dir is None: + destination_dir = os.path.join(sources_dir, "gyb_generated") + make_dir_if_needed(destination_dir) - if swiftsyntaxbuilder_destination is None: - swiftsyntaxbuilder_destination = os.path.join( - SWIFTSYNTAXBUILDER_DIR, "gyb_generated") + # Clear any *.swift files that are relics from the previous run. + clear_gyb_files_from_previous_run( + sources_dir, destination_dir, verbose) + + # Generate the new .swift files in `temp_files_dir` and only copy them + # to `destiantion_dir` if they are different than the + # files already residing there. This way we don't touch the generated .swift + # files if they haven't changed and don't trigger a rebuild. + for gyb_file in os.listdir(sources_dir): + if not gyb_file.endswith(".gyb"): + continue - template_destination = os.path.join(swiftsyntax_destination, "syntax_nodes") + gyb_file_path = os.path.join(sources_dir, gyb_file) + + # Slice off the '.gyb' to get the name for the output file + output_file_name = gyb_file[:-4] + + generate_single_gyb_file( + gyb_exec, + gyb_file_path, + output_file_name, + destination_dir, + temp_files_dir, + add_source_locations, + additional_gyb_flags=[], + verbose=verbose, + ) +# Generate the syntax node `.swift` files from `SyntaxNodes.swift.gyb.template`. +# `destination_dir` is not `None`, the resulting files will be written to +# `destination_dir/syntax_nodes`, otherwise they will be written to +# `sources_dir/gyb_generated/syntax_nodes`. +def generate_syntax_node_template_gyb_files( + destination_dir, + gyb_exec, + add_source_locations, + verbose +): + temp_files_dir = tempfile.gettempdir() make_dir_if_needed(temp_files_dir) - make_dir_if_needed(swiftsyntax_destination) - make_dir_if_needed(swiftsyntaxbuilder_destination) - make_dir_if_needed(template_destination) - # Clear any *.swift files that are relics from the previous run. - clear_gyb_files_from_previous_run( - SWIFTSYNTAX_DIR, swiftsyntax_destination, verbose) + if destination_dir is None: + destination_dir = os.path.join(SWIFTSYNTAX_DIR, "gyb_generated") - clear_gyb_files_from_previous_run( - SWIFTSYNTAXBUILDER_DIR, swiftsyntaxbuilder_destination, verbose) + template_destination = os.path.join(destination_dir, "syntax_nodes") + make_dir_if_needed(template_destination) for previous_gyb_gen_file in os.listdir(template_destination): if previous_gyb_gen_file.endswith(".swift"): if previous_gyb_gen_file not in BASE_KIND_FILES.values(): @@ -209,13 +241,6 @@ def generate_gyb_files( verbose=verbose, ) - generate_gyb_files_helper(gyb_exec, SWIFTSYNTAX_DIR, swiftsyntax_destination, - temp_files_dir, add_source_locations, verbose) - - generate_gyb_files_helper(gyb_exec, SWIFTSYNTAXBUILDER_DIR, - swiftsyntaxbuilder_destination, temp_files_dir, add_source_locations, - verbose) - for base_kind in BASE_KIND_FILES: output_file_name = BASE_KIND_FILES[base_kind] @@ -234,6 +259,45 @@ def generate_gyb_files( verbose=verbose, ) + +def generate_gyb_files( + gyb_exec, verbose, add_source_locations, + swiftsyntax_destination=None, swiftsyntaxbuilder_destination=None, + swiftsyntaxparser_destination=None, +): + print("** Generating gyb Files **") + + check_gyb_exec(gyb_exec) + check_rsync() + + generate_gyb_files_helper( + SWIFTSYNTAX_DIR, + swiftsyntax_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_gyb_files_helper( + SWIFTSYNTAXBUILDER_DIR, + swiftsyntaxbuilder_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_gyb_files_helper( + SWIFTSYNTAXPARSER_DIR, + swiftsyntaxparser_destination, + gyb_exec, + add_source_locations, + verbose + ) + generate_syntax_node_template_gyb_files( + swiftsyntax_destination, + gyb_exec, + add_source_locations, + verbose + ) + print("Done Generating gyb Files") @@ -257,40 +321,6 @@ def clear_gyb_files_from_previous_run(sources_dir, destination_dir, verbose): verbose=verbose ) - -# Generate the new .swift files in `temp_files_dir` and only copy them -# to `destiantion_dir` if they are different than the -# files already residing there. This way we don't touch the generated .swift -# files if they haven't changed and don't trigger a rebuild. -def generate_gyb_files_helper( - gyb_exec, - sources_dir, - destination_dir, - temp_files_dir, - add_source_locations, - verbose -): - for gyb_file in os.listdir(sources_dir): - if not gyb_file.endswith(".gyb"): - continue - - gyb_file_path = os.path.join(sources_dir, gyb_file) - - # Slice off the '.gyb' to get the name for the output file - output_file_name = gyb_file[:-4] - - generate_single_gyb_file( - gyb_exec, - gyb_file_path, - output_file_name, - destination_dir, - temp_files_dir, - add_source_locations, - additional_gyb_flags=[], - verbose=verbose, - ) - - # ----------------------------------------------------------------------------- # Building SwiftSyntax @@ -359,13 +389,16 @@ def verify_generated_files(gyb_exec, verbose): user_swiftsyntax_generated_dir = os.path.join( SWIFTSYNTAX_DIR, "gyb_generated" ) - user_swiftsyntaxbuilder_generated_dir = os.path.join( SWIFTSYNTAXBUILDER_DIR, "gyb_generated" ) + user_swiftsyntaxparser_generated_dir = os.path.join( + SWIFTSYNTAXPARSER_DIR, "gyb_generated" + ) self_swiftsyntax_generated_dir = tempfile.mkdtemp() self_swiftsyntaxbuilder_generated_dir = tempfile.mkdtemp() + self_swiftsyntaxparser_generated_dir = tempfile.mkdtemp() generate_gyb_files( gyb_exec, @@ -373,19 +406,22 @@ def verify_generated_files(gyb_exec, verbose): add_source_locations=False, swiftsyntax_destination=self_swiftsyntax_generated_dir, swiftsyntaxbuilder_destination=self_swiftsyntaxbuilder_generated_dir, + swiftsyntaxparser_destination=self_swiftsyntaxparser_generated_dir, ) check_generated_files_match(self_swiftsyntax_generated_dir, user_swiftsyntax_generated_dir) - check_generated_files_match(self_swiftsyntaxbuilder_generated_dir, + check_generated_files_match(self_swiftsyntaxbuilder_generated_dir, user_swiftsyntaxbuilder_generated_dir) + check_generated_files_match(self_swiftsyntaxparser_generated_dir, + user_swiftsyntaxparser_generated_dir) def check_generated_files_match(self_generated_dir, user_generated_dir): command = [ "diff", - "-r", - "-x", + "--recursive", + "--exclude", ".*", # Exclude dot files like .DS_Store "--context=0", self_generated_dir, @@ -394,6 +430,22 @@ def check_generated_files_match(self_generated_dir, user_generated_dir): check_call(command) +def verify_c_syntax_nodes_match(): + print("** Validating that the C data types match **") + + swift_syntax_c_definitions = os.path.join( + SOURCES_DIR, "_CSwiftSyntax", "include", "c-syntax-nodes.h") + swiftc_c_definitions = os.path.join( + SWIFT_DIR, "include", "swift-c", "SyntaxParser", + "SwiftSyntaxCDataTypes.h") + + check_call([ + "diff", + swift_syntax_c_definitions, + swiftc_c_definitions, + ]) + + def run_tests( toolchain, build_dir, multiroot_data_file, release, filecheck_exec, skip_lit_tests, verbose @@ -790,6 +842,8 @@ def main(): if args.degyb_only: sys.exit(0) + verify_c_syntax_nodes_match() + if args.generate_xcodeproj: xcode_gen(config=args.xcconfig_path) sys.exit(0) diff --git a/utils/group.json b/utils/group.json index c324df51afb..23c40d97423 100644 --- a/utils/group.json +++ b/utils/group.json @@ -32,14 +32,6 @@ "SyntaxVisitor.swift", "Misc.swift", ], - "Diagnostic": [ - "Diagnostic.swift", - "DiagnosticConsumer.swift", - "DiagnosticEngine.swift", - "JSONDiagnosticConsumer.swift", - "PrintingDiagnosticConsumer.swift", - "SyntaxVerifier.swift", - ], "Position": [ "AbsolutePosition.swift", "SourceLength.swift", @@ -48,9 +40,11 @@ ], "Parse": [ "IncrementalParseTransition.swift", - "SyntaxParser.swift", ], "Internal": [ + "_SyntaxParserInterop.swift", + "CNodes.swift", "AtomicCounter.swift", + "SyntaxVerifier.swift", ] }