Tweak the MSVC environment vars cache

CHANGES.txt

@@ -109,6 +109,15 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       processors for testing threads.
     - Fixed crash in C scanner's dictify_CPPDEFINES() function which happens if
       AppendUnique is called on CPPPATH. (Issue #4108).
+    - The MSVC script_env_cache now contains a sanity check: if the retrieved
+      tools path does not exist, the entry is invalidated so it will
+      be recomputed, in an attempt to avoid scons failing when certain
+      compiler version bumps have taken place.  The dictionary key (uses
+      the name of a batch file and any arguments which may have been
+      passes), is now computed a bit differently: the dashes are left
+      off if there are no arguments.  The default cachefile is changed
+      to have a .json suffix, for better recognition on Windows since
+      the contents are json.
     - As "code modernization" all of SCons now uses the current super()
       zero-argument syntax instead of direct calls to a parent class method
       or the super() two-argument syntax.

RELEASE.txt

@@ -30,9 +30,10 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
 ---------------------------------------
 
 - On Windows, %AllUsersProfile%\scons\site_scons is now the default "system"
-  location for a site_scons. %AllUsersProfile%\Application Data\scons\site_scons
-  will continue to work. There does not seem to be any convention to use
-  an "Application Data" subdirectory here.
+  location for a site_scons directory.
+  %AllUsersProfile%\Application Data\scons\site_scons will continue to work.
+  There does not seem to be any existing convention to use an
+  "Application Data" subdirectory here.
 - Action._subproc() can now be used as a python context manager to ensure that the
   POpen object is properly closed.
 - SCons help (-H) no longer prints the "ignored for compatibility" options,
@@ -45,6 +46,12 @@ CHANGED/ENHANCED EXISTING FUNCTIONALITY
 - The change to "content" and "content-timestamp" Decider names is reflected
   in the User Guide as well, since the hash function may be other than md5
   (tidying up from earlier change)
+- If SCONS_CACHE_MSVC_CONFIG is used, it will now attempt a sanity check for
+  the cached compiler information, and regenerate the information
+  if needed, rather than just failing after certain compiler version
+  changes have  happened.  The cache file can still be manually removed
+  if there are issues to force a regen.  The default cache filename now
+  has a .json suffix - the contents have always been json.
 - Update ninja file generation to only create response files for build commands
   which exceed MAXLINELENGTH
 - Update the debug output written to stdout for MSVC initialization which is enabled

SCons/Tool/MSCommon/common.py

@@ -31,6 +31,8 @@
 import re
 import subprocess
 import sys
+from contextlib import suppress
+from pathlib import Path
 
 import SCons.Util
 
@@ -94,16 +96,21 @@ def debug(x, *args):
 # SCONS_CACHE_MSVC_CONFIG is public, and is documented.
 CONFIG_CACHE = os.environ.get('SCONS_CACHE_MSVC_CONFIG')
 if CONFIG_CACHE in ('1', 'true', 'True'):
-    CONFIG_CACHE = os.path.join(os.path.expanduser('~'), '.scons_msvc_cache')
+    CONFIG_CACHE = os.path.join(os.path.expanduser('~'), 'scons_msvc_cache.json')
 
 
 def read_script_env_cache():
     """ fetch cached msvc env vars if requested, else return empty dict """
     envcache = {}
     if CONFIG_CACHE:
         try:
-            with open(CONFIG_CACHE, 'r') as f:
-                envcache = json.load(f)
+            p = Path(CONFIG_CACHE)
+            with p.open('r') as f:
+                # Convert the list of cache entry dictionaries read from
+                # json to the cache dictionary. Reconstruct the cache key
+                # tuple from the key list written to json.
+                envcache_list = json.load(f)
+                envcache = {tuple(d['key']): d['data'] for d in envcache_list}
         except FileNotFoundError:
             # don't fail if no cache file, just proceed without it
             pass
@@ -114,11 +121,17 @@ def write_script_env_cache(cache):
     """ write out cache of msvc env vars if requested """
     if CONFIG_CACHE:
         try:
-            with open(CONFIG_CACHE, 'w') as f:
-                json.dump(cache, f, indent=2)
+            p = Path(CONFIG_CACHE)
+            with p.open('w') as f:
+                # Convert the cache dictionary to a list of cache entry
+                # dictionaries. The cache key is converted from a tuple to
+                # a list for compatibility with json.
+                envcache_list = [{'key': list(key), 'data': data} for key, data in cache.items()]
+                json.dump(envcache_list, f, indent=2)
         except TypeError:
             # data can't serialize to json, don't leave partial file
-            os.remove(CONFIG_CACHE)
+            with suppress(FileNotFoundError):
+                p.unlink()
         except IOError:
             # can't write the file, just skip
             pass

SCons/Tool/MSCommon/vc.py

@@ -43,6 +43,7 @@
 import subprocess
 import os
 import platform
+from pathlib import Path
 from string import digits as string_digits
 from subprocess import PIPE
 import re
@@ -985,8 +986,25 @@ def script_env(script, args=None):
 
     if script_env_cache is None:
         script_env_cache = common.read_script_env_cache()
-    cache_key = "{}--{}".format(script, args)
+    cache_key = (script, args if args else None)
     cache_data = script_env_cache.get(cache_key, None)
+
+    # Brief sanity check: if we got a value for the key,
+    # see if it has a VCToolsInstallDir entry that is not empty.
+    # If so, and that path does not exist, invalidate the entry.
+    # If empty, this is an old compiler, just leave it alone.
+    if cache_data is not None:
+        try:
+            toolsdir = cache_data["VCToolsInstallDir"]
+        except KeyError:
+            # we write this value, so should not happen
+            pass
+        else:
+            if toolsdir:
+                toolpath = Path(toolsdir[0])
+                if not toolpath.exists():
+                    cache_data = None
+
     if cache_data is None:
         stdout = common.get_output(script, args)
 

doc/man/scons.xml

@@ -8155,7 +8155,7 @@ in tightly controlled Continuous Integration setups.</para>
 <para>If set to a True-like value (<literal>"1"</literal>,
 <literal>"true"</literal> or
 <literal>"True"</literal>) will cache to a file named
-<filename>.scons_msvc_cache</filename> in the user's home directory.
+<filename>.scons_msvc_cache.json</filename> in the user's home directory.
 If set to a pathname, will use that pathname for the cache.</para>
 
 <para>Note: use this cache with caution as it