Skip to content

Commit d64c263

Browse files
authored
Add a flag to parse serially (#1063)
* Don't parse in parallel Parsing in parallel is leading to unexpected `EXC_BAD_ACCESS`es in our codebase. By switching to `compactMap`, the crashes go away. The specific crash is deep in the internals of `SwiftSyntax`, which leads me to believe that there's some internal global thread-unsafe state that's being shared bewteen parsing instances. Here's an example (abbreviated) stacktrace: ``` #0 0x00000001001209e4 in protocol witness for SyntaxProtocol._syntaxNode.getter in conformance Syntax () #1 0x0000000100121ec3 in SyntaxProtocol.raw.getter at /Users/eric_horacek/Library/Developer/Xcode/DerivedData/Sourcery-hibchsafsxkdcjaxxaylrbeyoydh/SourcePackages/checkouts/swift-syntax/Sources/SwiftSyntax/Syntax.swift:129 #2 0x0000000100b85c32 in SimpleTypeIdentifierSyntax.init(_:) at /Users/eric_horacek/Library/Developer/Xcode/DerivedData/Sourcery-hibchsafsxkdcjaxxaylrbeyoydh/SourcePackages/checkouts/swift-syntax/Sources/SwiftSyntax/gyb_generated/syntax_nodes/SyntaxTypeNodes.swift:68 #3 0x0000000100b89979 in protocol witness for SyntaxProtocol.init(_:) in conformance SimpleTypeIdentifierSyntax () #4 0x00000001001acab7 in TypeSyntax.as<τ_0_0>(_:) at /Users/eric_horacek/Library/Developer/Xcode/DerivedData/Sourcery-hibchsafsxkdcjaxxaylrbeyoydh/SourcePackages/checkouts/swift-syntax/Sources/SwiftSyntax/gyb_generated/SyntaxBaseNodes.swift:421 ``` I was able to reproduce this in the debugger, but there was no information available with either TSAN or ASAN. Fixes #1009. I know that a lot of folks rely on the performance of parsing in parallel and it only seems to be crashing on our codebase, so perhaps we could put this behind an argument? * Add back OSAtomicIncrement32 for numberOfFilesThatHadToBeParsed * Add a configuration flag for parsing in serial * Add changelog entry
1 parent 9616fad commit d64c263

File tree

3 files changed

+19
-4
lines changed

3 files changed

+19
-4
lines changed

CHANGELOG.md

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
# Sourcery CHANGELOG
22

33
## 1.8.1
4+
## New Features
5+
- Added a new flag `--serialParse` to support parsing the sources in serial, rather than in parallel (the default), which can address stability issues in SwiftSyntax [#1063](https://github.com/krzysztofzablocki/Sourcery/pull/1063)
6+
47
## Internal Changes
58
- Lower project requirements to allow compilation using Swift 5.5/Xcode 13.x [#1049](https://github.com/krzysztofzablocki/Sourcery/pull/1049)
69
- Update Stencil to 0.14.2

Sourcery/Sourcery.swift

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ public class Sourcery {
3030
fileprivate let cacheDisabled: Bool
3131
fileprivate let cacheBasePath: Path?
3232
fileprivate let prune: Bool
33+
fileprivate let serialParse: Bool
3334

3435
fileprivate var status = ""
3536
fileprivate var templatesPaths = Paths(include: [])
@@ -44,13 +45,14 @@ public class Sourcery {
4445
}
4546

4647
/// Creates Sourcery processor
47-
public init(verbose: Bool = false, watcherEnabled: Bool = false, cacheDisabled: Bool = false, cacheBasePath: Path? = nil, prune: Bool = false, arguments: [String: NSObject] = [:]) {
48+
public init(verbose: Bool = false, watcherEnabled: Bool = false, cacheDisabled: Bool = false, cacheBasePath: Path? = nil, prune: Bool = false, serialParse: Bool = false, arguments: [String: NSObject] = [:]) {
4849
self.verbose = verbose
4950
self.arguments = arguments
5051
self.watcherEnabled = watcherEnabled
5152
self.cacheDisabled = cacheDisabled
5253
self.cacheBasePath = cacheBasePath
5354
self.prune = prune
55+
self.serialParse = serialParse
5456
}
5557

5658
/// Processes source files and generates corresponding code.
@@ -309,16 +311,24 @@ extension Sourcery {
309311
numberOfFilesThatHadToBeParsed = 0
310312

311313
var lastError: Swift.Error?
312-
let results = parserGenerator.parallelCompactMap { parser -> FileParserResult? in
314+
315+
let transform: (ParserWrapper) -> FileParserResult? = { parser in
313316
do {
314-
return try self.loadOrParse(parser: parser, cachesPath: cachesDir(sourcePath: from))
317+
return try self.loadOrParse(parser: parser, cachesPath: self.cachesDir(sourcePath: from))
315318
} catch {
316319
lastError = error
317320
Log.error("Unable to parse \(parser.path), error \(error)")
318321
return nil
319322
}
320323
}
321324

325+
let results: [FileParserResult]
326+
if serialParse {
327+
results = parserGenerator.compactMap(transform)
328+
} else {
329+
results = parserGenerator.parallelCompactMap(transform: transform)
330+
}
331+
322332
if let error = lastError {
323333
throw error
324334
}

SourceryExecutable/main.swift

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@ func runCLI() {
9696
Flag("parseDocumentation", description: "Include documentation comments for all declarations."),
9797
Flag("quiet", flag: "q", description: "Turn off any logging, only emmit errors."),
9898
Flag("prune", flag: "p", description: "Remove empty generated files"),
99+
Flag("serialParse", description: "Parses the specified sources in serial, rather than in parallel (the default), which can address stability issues in SwiftSyntax."),
99100
VariadicOption<Path>("sources", description: "Path to a source swift files. File or Directory."),
100101
VariadicOption<Path>("exclude-sources", description: "Path to a source swift files to exclude. File or Directory."),
101102
VariadicOption<Path>("templates", description: "Path to templates. File or Directory."),
@@ -111,7 +112,7 @@ func runCLI() {
111112
via `argument.<name>`. To pass in string you should use escaped quotes (\\").
112113
"""),
113114
Option<Path>("ejsPath", default: "", description: "Path to EJS file for JavaScript templates.")
114-
) { watcherEnabled, disableCache, verboseLogging, logAST, logBenchmark, parseDocumentation, quiet, prune, sources, excludeSources, templates, excludeTemplates, output, configPaths, forceParse, args, ejsPath in
115+
) { watcherEnabled, disableCache, verboseLogging, logAST, logBenchmark, parseDocumentation, quiet, prune, serialParse, sources, excludeSources, templates, excludeTemplates, output, configPaths, forceParse, args, ejsPath in
115116
do {
116117
switch (quiet, verboseLogging) {
117118
case (true, _):
@@ -192,6 +193,7 @@ func runCLI() {
192193
cacheDisabled: disableCache,
193194
cacheBasePath: configuration.cacheBasePath,
194195
prune: prune,
196+
serialParse: serialParse,
195197
arguments: configuration.args)
196198

197199
return try sourcery.processFiles(

0 commit comments

Comments
 (0)