diff --git a/.github/workflows/mad_modelDiff.yml b/.github/workflows/mad_modelDiff.yml index 3d98e74f0fbb..b0e4a20f2b84 100644 --- a/.github/workflows/mad_modelDiff.yml +++ b/.github/workflows/mad_modelDiff.yml @@ -68,7 +68,7 @@ jobs: DATABASE=$2 cd codeql-$QL_VARIANT SHORTNAME=`basename $DATABASE` - python java/ql/src/utils/modelgenerator/GenerateFlowModel.py --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT + python misc/scripts/models-as-data/generate_mad.py --language java --with-summaries --with-sinks $DATABASE $SHORTNAME/$QL_VARIANT mkdir -p $MODELS/$SHORTNAME mv java/ql/lib/ext/generated/$SHORTNAME/$QL_VARIANT $MODELS/$SHORTNAME cd .. diff --git a/cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py b/cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py deleted file mode 100644 index 38bbaad118bd..000000000000 --- a/cpp/ql/src/utils/modelgenerator/GenerateFlowModel.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import sys -import os.path -import subprocess - -# Add Model as Data script directory to sys.path. -gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() -madpath = os.path.join(gitroot, "misc/scripts/models-as-data/") -sys.path.append(madpath) - -import generate_flow_model as model - -language = "cpp" -model.Generator.make(language).run() diff --git a/csharp/ql/src/change-notes/2025-05-02-mad-generator-renamed.md b/csharp/ql/src/change-notes/2025-05-02-mad-generator-renamed.md new file mode 100644 index 000000000000..824b37180a18 --- /dev/null +++ b/csharp/ql/src/change-notes/2025-05-02-mad-generator-renamed.md @@ -0,0 +1,8 @@ +--- +category: minorAnalysis +--- +* Changes to the MaD model generation infrastructure: + * The `csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has + been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now + supports being called directly and should be used instead. The script + requires a `--language` argument but otherwise functions identically. diff --git a/csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py b/csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py deleted file mode 100755 index ebc7ea1be789..000000000000 --- a/csharp/ql/src/utils/modelgenerator/GenerateFlowModel.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import sys -import os.path -import subprocess - -# Add Model as Data script directory to sys.path. -gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() -madpath = os.path.join(gitroot, "misc/scripts/models-as-data/") -sys.path.append(madpath) - -import generate_flow_model as model - -language = "csharp" -model.Generator.make(language).run() \ No newline at end of file diff --git a/java/ql/src/change-notes/2025-05-02-mad-generator-renamed.md b/java/ql/src/change-notes/2025-05-02-mad-generator-renamed.md new file mode 100644 index 000000000000..0a2d9e797fc5 --- /dev/null +++ b/java/ql/src/change-notes/2025-05-02-mad-generator-renamed.md @@ -0,0 +1,8 @@ +--- +category: minorAnalysis +--- +* Changes to the MaD model generation infrastructure: + * The `java/ql/src/utils/modelgenerator/GenerateFlowModel.py` script has + been removed. The `/misc/scripts/models-as-data/generate_mad.py` script now + supports being called directly and should be used instead. The script + requires a `--language` argument but otherwise functions identically. diff --git a/java/ql/src/utils/modelgenerator/GenerateFlowModel.py b/java/ql/src/utils/modelgenerator/GenerateFlowModel.py deleted file mode 100755 index 44b40a4d34a4..000000000000 --- a/java/ql/src/utils/modelgenerator/GenerateFlowModel.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import sys -import os.path -import subprocess - -# Add Model as Data script directory to sys.path. -gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() -madpath = os.path.join(gitroot, "misc/scripts/models-as-data/") -sys.path.append(madpath) - -import generate_flow_model as model - -language = "java" -model.Generator.make(language).run() \ No newline at end of file diff --git a/java/ql/src/utils/modelgenerator/RegenerateModels.py b/java/ql/src/utils/modelgenerator/RegenerateModels.py index 1f7df71776b1..0c38f41c0694 100755 --- a/java/ql/src/utils/modelgenerator/RegenerateModels.py +++ b/java/ql/src/utils/modelgenerator/RegenerateModels.py @@ -35,8 +35,8 @@ def regenerateModel(lgtmSlug, extractedDb): sys.exit(1) modelFile = lgtmSlugToModelFile[lgtmSlug] codeQlRoot = findGitRoot() - subprocess.check_call([codeQlRoot + "/java/ql/src/utils/modelgenerator/GenerateFlowModel.py", - "--with-summaries", "--with-sinks", "--with-neutrals", + subprocess.check_call([codeQlRoot + "/misc/scripts/models-as-data/generate_mad.py", + "--language", "java", "--with-summaries", "--with-sinks", "--with-neutrals", extractedDb, modelFile]) print("Regenerated " + modelFile) shutil.rmtree(tmpDir) diff --git a/misc/scripts/models-as-data/generate_flow_model.py b/misc/scripts/models-as-data/generate_mad.py similarity index 86% rename from misc/scripts/models-as-data/generate_flow_model.py rename to misc/scripts/models-as-data/generate_mad.py index 1d108015fa0b..60674ed54eda 100755 --- a/misc/scripts/models-as-data/generate_flow_model.py +++ b/misc/scripts/models-as-data/generate_mad.py @@ -1,10 +1,8 @@ #!/usr/bin/python3 import helpers -import json import os import os.path -import shlex import subprocess import sys import tempfile @@ -27,24 +25,13 @@ def parseData(data): return rows -class Generator: - def __init__ (self, language): - self.language = language - self.generateSinks = False - self.generateSources = False - self.generateSummaries = False - self.generateNeutrals = False - self.generateTypeBasedSummaries = False - self.dryRun = False - self.dirname = "modelgenerator" - - def printHelp(self): - print(f"""Usage: -python3 GenerateFlowModel.py [DIR] [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run] +def printHelp(): + print(f"""Usage: +python3 generate_mad.py [DIR] --language LANGUAGE [--with-sinks] [--with-sources] [--with-summaries] [--with-neutrals] [--with-typebased-summaries] [--dry-run] This generates summary, source, sink and neutral models for the code in the database. -The files will be placed in `{self.language}/ql/lib/ext/generated/DIR` +The files will be placed in `LANGUAGE/ql/lib/ext/generated/DIR` Which models are generated is controlled by the flags: --with-sinks @@ -57,14 +44,25 @@ def printHelp(self): --dry-run: Only run the queries, but don't write to file. Example invocations: -$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db -$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks -$ python3 GenerateFlowModel.py /tmp/dbs/my_library_db --with-sinks my_directory +$ python3 generate_mad.py /tmp/dbs/my_library_db +$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks +$ python3 generate_mad.py /tmp/dbs/my_library_db --with-sinks my_directory Requirements: `codeql` should appear on your path. """) +class Generator: + def __init__(self, language): + self.language = language + self.generateSinks = False + self.generateSources = False + self.generateSummaries = False + self.generateNeutrals = False + self.generateTypeBasedSummaries = False + self.dryRun = False + self.dirname = "modelgenerator" + def setenvironment(self, database, folder): self.codeQlRoot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() @@ -76,12 +74,22 @@ def setenvironment(self, database, folder): @staticmethod - def make(language): - generator = Generator(language) + def make(): + '''Create a generator instance based on command line arguments.''' if any(s == "--help" for s in sys.argv): - generator.printHelp() + printHelp() + sys.exit(0) + + if "--language" in sys.argv: + language = sys.argv[sys.argv.index("--language") + 1] + sys.argv.remove("--language") + sys.argv.remove(language) + else: + print("Error: Language not specified. Use --language .") sys.exit(0) + generator = Generator(language=language) + if "--with-sinks" in sys.argv: sys.argv.remove("--with-sinks") generator.generateSinks = True @@ -115,7 +123,7 @@ def make(language): n = len(sys.argv) if n < 2: - generator.printHelp() + printHelp() sys.exit(1) elif n == 2: generator.setenvironment(sys.argv[1], "") @@ -204,3 +212,6 @@ def run(self): if self.generateTypeBasedSummaries: self.save(typeBasedContent, ".typebased.model.yml") + +if __name__ == '__main__': + Generator.make().run() \ No newline at end of file diff --git a/rust/ql/src/utils/modelgenerator/GenerateFlowModel.py b/rust/ql/src/utils/modelgenerator/GenerateFlowModel.py deleted file mode 100644 index 008b82501ff9..000000000000 --- a/rust/ql/src/utils/modelgenerator/GenerateFlowModel.py +++ /dev/null @@ -1,15 +0,0 @@ -#!/usr/bin/python3 - -import sys -import os.path -import subprocess - -# Add Model as Data script directory to sys.path. -gitroot = subprocess.check_output(["git", "rev-parse", "--show-toplevel"]).decode("utf-8").strip() -madpath = os.path.join(gitroot, "misc/scripts/models-as-data/") -sys.path.append(madpath) - -import generate_flow_model as model - -language = "rust" -model.Generator.make(language).run()