Improvements to lex and yacc tools

CHANGES.txt

@@ -202,6 +202,8 @@ RELEASE  VERSION/DATE TO BE FILLED IN LATER
       but was not applied to other dialects, and e2e tests explicitly checked
       that FORTRANFLAGS did not propagate outside the FORTRAN dialect,
       so the conclusion is that behavior is intentional (issue #2257)
+    - Improvements to lex and yacc tools: better documentation of
+      extra-file options, add test for extra-file behavior.
 
   From Zhichang Yu:
     - Added MSVC_USE_SCRIPT_ARGS variable to pass arguments to MSVC_USE_SCRIPT.

SCons/Tool/__init__.py

@@ -36,6 +36,7 @@
 import sys
 import os
 import importlib.util
+from typing import Optional
 
 import SCons.Builder
 import SCons.Errors
@@ -829,7 +830,7 @@ def tool_list(platform, env):
     return [x for x in tools if x]
 
 
-def find_program_path(env, key_program, default_paths=None, add_path=False) -> str:
+def find_program_path(env, key_program, default_paths=None, add_path=False) -> Optional[str]:
     """
     Find the location of a tool using various means.
 
@@ -849,6 +850,8 @@ def find_program_path(env, key_program, default_paths=None, add_path=False) -> s
     # Then in the OS path
     path = SCons.Util.WhereIs(key_program)
     if path:
+        if add_path:
+            env.AppendENVPath('PATH', os.path.dirname(path))
         return path
 
     # Finally, add the defaults and check again.
@@ -864,8 +867,7 @@ def find_program_path(env, key_program, default_paths=None, add_path=False) -> s
     # leave that to the caller, unless add_path is true.
     env['ENV']['PATH'] = save_path
     if path and add_path:
-        bin_dir = os.path.dirname(path)
-        env.AppendENVPath('PATH', bin_dir)
+        env.AppendENVPath('PATH', os.path.dirname(path))
 
     return path
 

SCons/Tool/lex.py

@@ -23,21 +23,27 @@
 
 """Tool-specific initialization for lex.
 
+This tool should support multiple lex implementations,
+but is in actuality biased towards GNU Flex.
+
 There normally shouldn't be any need to import this module directly.
 It will usually be imported through the generic SCons.Tool.Tool()
 selection method.
 """
 
 import os.path
 import sys
+from typing import Optional
 
 import SCons.Action
 import SCons.Tool
-import SCons.Util
 import SCons.Warnings
 from SCons.Platform.mingw import MINGW_DEFAULT_PATHS
 from SCons.Platform.cygwin import CYGWIN_DEFAULT_PATHS
 from SCons.Platform.win32 import CHOCO_DEFAULT_PATH
+from SCons.Util import CLVar, to_String
+
+DEFAULT_PATHS = CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS
 
 LexAction = SCons.Action.Action("$LEXCOM", "$LEXCOMSTR")
 
@@ -47,49 +53,55 @@
     BINS = ["flex", "lex"]
 
 
-def lexEmitter(target, source, env):
-    sourceBase, sourceExt = os.path.splitext(SCons.Util.to_String(source[0]))
+def lexEmitter(target, source, env) -> tuple:
+    """Adds extra files generated by lex program to target list."""
 
+    sourceBase, sourceExt = os.path.splitext(to_String(source[0]))
     if sourceExt == ".lm":           # If using Objective-C
         target = [sourceBase + ".m"] # the extension is ".m".
 
-    # This emitter essentially tries to add to the target all extra
-    # files generated by flex.
-
-    # Different options that are used to trigger the creation of extra files.
+    # With --header-file and ----tables-file, the file to write is defined
+    # by the option argument. Extract this and include in the list of targets.
+    # NOTE: a filename passed to the command this way is not modified by SCons,
+    # and so will be interpreted relative to the project top directory at
+    # execution time, while the name added to the target list will be
+    # interpreted relative to the SConscript directory - a possible mismatch.
+    #
+    # These are GNU flex-only options.
+    # TODO: recognize --outfile also?
     file_gen_options = ["--header-file=", "--tables-file="]
-
     lexflags = env.subst_list("$LEXFLAGS", target=target, source=source)
-    for option in SCons.Util.CLVar(lexflags):
+    for option in lexflags[0]:
         for fileGenOption in file_gen_options:
             l = len(fileGenOption)
             if option[:l] == fileGenOption:
                 # A file generating option is present, so add the
                 # file name to the target list.
                 file_name = option[l:].strip()
                 target.append(file_name)
+
     return target, source
 
 
-def get_lex_path(env, append_paths=False):
+def get_lex_path(env, append_paths=False) -> Optional[str]:
     """
-    Find the path to the lex tool, searching several possible names
+    Returns the path to the lex tool, searching several possible names.
 
-    Only called in the Windows case, so the default_path
-    can be Windows-specific
+    Only called in the Windows case, so the `default_path` argument to
+    :func:`find_program_path` can be Windows-specific.
 
-    :param env: current construction environment
-    :param append_paths: if set, add the path to the tool to PATH
-    :return: path to lex tool, if found
+    Args:
+        env: current construction environment
+        append_paths: if set, add the path to the tool to PATH
     """
     for prog in BINS:
         bin_path = SCons.Tool.find_program_path(
             env,
             prog,
-            default_paths=CHOCO_DEFAULT_PATH + MINGW_DEFAULT_PATHS + CYGWIN_DEFAULT_PATHS )
+            default_paths=DEFAULT_PATHS,
+            add_path=append_paths,
+        )
         if bin_path:
-            if append_paths:
-                env.AppendENVPath('PATH', os.path.dirname(bin_path))
             return bin_path
 
     SCons.Warnings.warn(
@@ -98,7 +110,7 @@ def get_lex_path(env, append_paths=False):
     )
 
 
-def generate(env):
+def generate(env) -> None:
     """Add Builders and construction variables for lex to an Environment."""
     c_file, cxx_file = SCons.Tool.createCFileBuilders(env)
 
@@ -117,21 +129,21 @@ def generate(env):
     cxx_file.add_action(".ll", LexAction)
     cxx_file.add_emitter(".ll", lexEmitter)
 
-    env["LEXFLAGS"] = SCons.Util.CLVar("")
+    env["LEXFLAGS"] = CLVar("")
 
     if sys.platform == 'win32':
         # ignore the return - we do not need the full path here
         _ = get_lex_path(env, append_paths=True)
         env["LEX"] = env.Detect(BINS)
         if not env.get("LEXUNISTD"):
-            env["LEXUNISTD"] = SCons.Util.CLVar("")
+            env["LEXUNISTD"] = CLVar("")
         env["LEXCOM"] = "$LEX $LEXUNISTD $LEXFLAGS -t $SOURCES > $TARGET"
     else:
         env["LEX"] = env.Detect(BINS)
         env["LEXCOM"] = "$LEX $LEXFLAGS -t $SOURCES > $TARGET"
 
 
-def exists(env):
+def exists(env) -> Optional[str]:
     if sys.platform == 'win32':
         return get_lex_path(env)
     else:

SCons/Tool/lex.xml

@@ -1,6 +1,28 @@
 <?xml version="1.0"?>
 <!--
-__COPYRIGHT__
+ MIT License
+
+ Copyright The SCons Foundation
+
+ Permission is hereby granted, free of charge, to any person obtaining
+ a copy of this software and associated documentation files (the
+ "Software"), to deal in the Software without restriction, including
+ without limitation the rights to use, copy, modify, merge, publish,
+ distribute, sublicense, and/or sell copies of the Software, and to
+ permit persons to whom the Software is furnished to do so, subject to
+ the following conditions:
+
+ The above copyright notice and this permission notice shall be included
+ in all copies or substantial portions of the Software.
+
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY
+ KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+ WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+ LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+ OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
 
 This file is processed by the bin/SConsDoc.py module.
 See its __doc__ string for a discussion of the format.
@@ -37,6 +59,7 @@ Sets construction variables for the &lex; lexical analyser.
 </sets>
 <uses>
 <item>LEXCOMSTR</item>
+<item>LEXFLAGS</item>
 </uses>
 </tool>
 
@@ -66,7 +89,7 @@ If this is not set, then &cv-link-LEXCOM; (the command line) is displayed.
 </para>
 
 <example_commands>
-env = Environment(LEXCOMSTR = "Lex'ing $TARGET from $SOURCES")
+env = Environment(LEXCOMSTR="Lex'ing $TARGET from $SOURCES")
 </example_commands>
 </summary>
 </cvar>
@@ -75,6 +98,14 @@ env = Environment(LEXCOMSTR = "Lex'ing $TARGET from $SOURCES")
 <summary>
 <para>
 General options passed to the lexical analyzer generator.
+In addition to passing the value on during invocation,
+the &t-link-lex; tool also examines this &consvar; for options
+which cause additional output files to be generated,
+and adds those to the target list.
+Recognized for this purpose are GNU &flex; options
+<option>--header-file=</option> and
+<option>--tables-file=</option>;
+the output file is named by the option argument.
 </para>
 </summary>
 </cvar>