From 14882d6b7440e797d8d60cb1f8207fe332cae033 Mon Sep 17 00:00:00 2001 From: Alexander Richardson Date: Mon, 18 Sep 2023 09:08:24 -0700 Subject: [PATCH] [libc++][lit] Atomically update the persistent cache (#66538) When running multiple shards in parallel, one shard might write to the cache while another one is reading this cache. Instead of updating the file in place, write to a temporary file and swap the cache file using os.replace(). This is an atomic operation and means shards will either see the old state or the new one. --- libcxx/utils/libcxx/test/dsl.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/libcxx/utils/libcxx/test/dsl.py b/libcxx/utils/libcxx/test/dsl.py index 64e44ee0f765..d2e0c6036dd3 100644 --- a/libcxx/utils/libcxx/test/dsl.py +++ b/libcxx/utils/libcxx/test/dsl.py @@ -69,8 +69,14 @@ def _memoizeExpensiveOperation(extractCacheKey): if cacheKey not in cache: cache[cacheKey] = function(config, *args, **kwargs) # Update the persistent cache so it knows about the new key - with open(persistentCache, "wb") as cacheFile: + # We write to a PID-suffixed file and rename the result to + # ensure that the cache is not corrupted when running the test + # suite with multiple shards. Since this file is in the same + # directory as the destination, os.replace() will be atomic. + unique_suffix = ".tmp." + str(os.getpid()) + with open(persistentCache + unique_suffix, "wb") as cacheFile: pickle.dump(cache, cacheFile) + os.replace(persistentCache + unique_suffix, persistentCache) return cache[cacheKey] return f