From 5b36d6c269245f56b7b7ac2e1a61c9bbc5108485 Mon Sep 17 00:00:00 2001 From: Alex Richardson Date: Fri, 15 Sep 2023 12:07:39 -0700 Subject: [PATCH] [libc++][lit] Atomically update the persistent cache When running multiple shards in parallel, one shard might write to the cache while another one is reading this cache. Instead of updating the file in place, write to a temporary file and swap the cache file using os.replace(). This is an atomic operation and means shards will either see the old state or the new one. --- libcxx/utils/libcxx/test/dsl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py index 847cebf5962f6..7b58310542e13 100644 --- a/libcxx/utils/libcxx/test/dsl.py +++ b/libcxx/utils/libcxx/test/dsl.py @@ -69,8 +69,14 @@ def f(config, *args, **kwargs): if cacheKey not in cache: cache[cacheKey] = function(config, *args, **kwargs) # Update the persistent cache so it knows about the new key - with open(persistentCache, "wb") as cacheFile: + # We write to a PID-suffixed file and rename the result to + # ensure that the cache is not corrupted when running the test + # suite with multiple shards. Since this file is in the same + # directory as the destination, os.replace() will be atomic. + unique_suffix = ".tmp." + str(os.getpid()) + with open(persistentCache + unique_suffix, "wb") as cacheFile: pickle.dump(cache, cacheFile) + os.replace(persistentCache + unique_suffix, persistentCache) return cache[cacheKey] return f