1+ #! /usr/bin/env bash
2+
3+ # Creates a Markdown report that contains all results of all the anomaly detection methods.
4+ # It requires an already running Neo4j graph database with already scanned and analyzed artifacts.
5+ # The results will be written into the sub directory reports/anomaly-detection.
6+
7+ # Note that "scripts/prepareAnalysis.sh" is required to run prior to this script.
8+ # Note that either "anomalyDetectionCsv.sh" or "anomalyDetectionPython.sh" is required to run prior to this script.
9+
10+ # Requires executeQueryFunctions.sh, cleanupAfterReportGeneration.sh
11+
12+ # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
13+ set -o errexit -o pipefail
14+
15+ # Overrideable Constants (defaults also defined in sub scripts)
16+ REPORTS_DIRECTORY=${REPORTS_DIRECTORY:- " reports" }
17+ MARKDOWN_INCLUDES_DIRECTORY=${MARKDOWN_INCLUDES_DIRECTORY:- " includes" }
18+
19+ # # Get this "domains/anomaly-detection/summary" directory if not already set
20+ # Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution.
21+ # CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
22+ # This way non-standard tools like readlink aren't needed.
23+ ANOMALY_DETECTION_SUMMARY_DIR=${ANOMALY_DETECTION_SUMMARY_DIR:- $(CDPATH=. cd -- " $( dirname -- " ${BASH_SOURCE[0]} " ) " && pwd -P)}
24+ # echo "anomalyDetectionSummary: ANOMALY_DETECTION_SUMMARY_DIR=${ANOMALY_DETECTION_SUMMARY_DIR}"
25+ # Get the "scripts" directory by taking the path of this script and going one directory up.
26+ SCRIPTS_DIR=${SCRIPTS_DIR:- " ${ANOMALY_DETECTION_SUMMARY_DIR} /../../../scripts" } # Repository directory containing the shell scripts
27+
28+ MARKDOWN_INCLUDES_DIRECTORY=" includes"
29+ MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:- " ${SCRIPTS_DIR} /markdown" }
30+ # echo "anomalyDetectionSummary: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2
31+
32+ # Define functions to execute a cypher query from within a given file (first and only argument) like "execute_cypher" and "execute_cypher_summarized"
33+ source " ${SCRIPTS_DIR} /executeQueryFunctions.sh"
34+
35+ # Appends a Markdown table to an existing file and
36+ # removes redundant header + separator rows.
37+ #
38+ # Usage:
39+ # cat newTable.md | append_table myMarkdownFile.md
40+ #
41+ # append_table myMarkdownFile.md <<'EOF'
42+ # | Name | Score | Archetype |
43+ # | --- | --- | --- |
44+ # | Bar | 0.9 | Something |
45+ # EOF
46+ #
47+ # Behavior:
48+ # - Keeps the first header row and its following separator row.
49+ # - Removes all subsequent duplicate header + separator pairs.
50+ # - Leaves all data rows untouched.
51+ append_to_markdown_table () {
52+ local file=" $1 "
53+
54+ # Append stdin to the target file
55+ cat >> " ${file} "
56+
57+ # Clean up duplicate headers (header row + --- row)
58+ awk ' !seen[$0]++ || NR <= 2' " ${file} " > " ${file} .tmp" && mv " ${file} .tmp" " ${file} "
59+ }
60+
61+ # Run the anomaly detection main report generation.
62+ anomaly_detection_report_first_section () {
63+ local report_markdown_includes_directory=" ${FULL_REPORT_DIRECTORY} /${MARKDOWN_INCLUDES_DIRECTORY} "
64+ mkdir -p " ${report_markdown_includes_directory} "
65+
66+ execute_cypher " ${ANOMALY_DETECTION_SUMMARY_DIR} /AnomaliesPerAbstractionLayer.cypher" --output-markdown-table > " ${report_markdown_includes_directory} /AnomaliesPerAbstractionLayer.md"
67+ execute_cypher " ${ANOMALY_DETECTION_SUMMARY_DIR} /AnomaliesInTotal.cypher" --output-markdown-table > " ${report_markdown_includes_directory} /AnomaliesInTotal.md"
68+ }
69+
70+ # Aggregates all results in a Markdown report.
71+ #
72+ # Required Parameters:
73+ # - projection_node_label=...
74+ # Label of the nodes that will be used for the projection. Example: "Package"
75+ # - projection_language=...
76+ # Name of the associated programming language. Examples: "Java", "Typescript"
77+ anomaly_detection_deep_dive_report () {
78+ local nodeLabel
79+ nodeLabel=$( extractQueryParameter " projection_node_label" " ${@ } " )
80+
81+ local language
82+ language=$( extractQueryParameter " projection_language" " ${@ } " )
83+
84+ local report_number
85+ report_number=$( extractQueryParameter " report_number" " ${@ } " )
86+
87+ echo " anomalyDetectionSummary: $( date +' %Y-%m-%dT%H:%M:%S%z' ) Creating ${language} ${nodeLabel} anomaly summary Markdown report..."
88+
89+ detail_report_directory_name=" ${language} _${nodeLabel} "
90+ detail_report_directory=" ${FULL_REPORT_DIRECTORY} /${detail_report_directory_name} "
91+ detail_report_include_directory=" ${detail_report_directory} /${MARKDOWN_INCLUDES_DIRECTORY} "
92+ mkdir -p " ${detail_report_include_directory} "
93+
94+ # Collect dynamic Markdown includes
95+ execute_cypher " ${ANOMALY_DETECTION_SUMMARY_DIR} /AnomaliesDeepDiveOverview.cypher" " ${@ } " --output-markdown-table > " ${detail_report_include_directory} /DeepDiveOverview.md"
96+ execute_cypher " ${ANOMALY_DETECTION_SUMMARY_DIR} /AnomaliesDeepDiveArchetypes.cypher" " ${@ } " --output-markdown-table > " ${detail_report_include_directory} /DeepDiveArchetypes.md"
97+ execute_cypher " ${ANOMALY_DETECTION_SUMMARY_DIR} /AnomalyDeepDiveTopAnomalies.cypher" " ${@ } " --output-markdown-table > " ${detail_report_include_directory} /DeepDiveTopAnomalies.md"
98+
99+ # Remove empty Markdown includes
100+ source " ${SCRIPTS_DIR} /cleanupAfterReportGeneration.sh" " ${detail_report_include_directory} "
101+
102+ # Collect static Markdown includes (after cleanup to not remove one-liner)
103+ echo " ### 2.${report_number} ${language} ${nodeLabel} " > " ${detail_report_include_directory} /DeepDiveSectionTitle.md"
104+ echo " " > " ${detail_report_include_directory} /empty.md"
105+ cp -f " ${ANOMALY_DETECTION_SUMMARY_DIR} /report_no_data_info.template.md" " ${detail_report_include_directory} /report_no_data_info.template.md"
106+ cp -f " ${detail_report_directory} /Top_anomaly_features.md" " ${detail_report_include_directory} " || true
107+
108+ if [ -f " ${detail_report_directory} /Anomaly_feature_importance_explained.svg" ] ; then
109+ cp -f " ${ANOMALY_DETECTION_SUMMARY_DIR} /report_deep_dive_anomalies_explained.template.md" " ${detail_report_include_directory} /report_deep_dive_anomalies_explained.md"
110+ fi
111+
112+ # Use Markdown template to assemble the final deep dive section of the Markdown report and replace variables
113+ cp -f " ${ANOMALY_DETECTION_SUMMARY_DIR} /report_deep_dive.template.md" " ${detail_report_directory} /report_deep_dive.template.md"
114+ cat " ${detail_report_directory} /report_deep_dive.template.md" | " ${MARKDOWN_SCRIPTS_DIR} /embedMarkdownIncludes.sh" " ${detail_report_include_directory} " > " ${detail_report_directory} /report_deep_dive_with_vars.md"
115+ sed " s/{{deep_dive_directory}}/${detail_report_directory_name} /g" " ${detail_report_directory} /report_deep_dive_with_vars.md" > " ${detail_report_directory} /report_deep_dive_${report_number} .md"
116+
117+ rm -rf " ${detail_report_directory} /report_deep_dive_with_vars.md"
118+ rm -rf " ${detail_report_directory} /report_deep_dive.template.md"
119+ rm -rf " ${detail_report_include_directory} "
120+
121+ # Clean-up after report generation. Empty reports will be deleted.
122+ source " ${SCRIPTS_DIR} /cleanupAfterReportGeneration.sh" " ${detail_report_directory} "
123+ }
124+
125+ # Run the anomaly detection report generation.
126+ #
127+ # Required Parameters:
128+ # - projection_node_label=...
129+ # Label of the nodes that will be used for the projection. Example: "Package"
130+ # - projection_language=...
131+ # Name of the associated programming language. Examples: "Java", "Typescript"
132+ anomaly_detection_report () {
133+ time anomaly_detection_deep_dive_report " ${@ } "
134+ }
135+
136+ # Finalize the anomaly detection report by taking the main template, applying includes and appending all deep dive reports
137+ anomaly_detection_finalize_report () {
138+ echo " anomalyDetectionSummary: $( date +' %Y-%m-%dT%H:%M:%S%z' ) Finalizing ${language} ${nodeLabel} anomaly detection Markdown report..."
139+
140+ report_include_directory=" ${FULL_REPORT_DIRECTORY} /${MARKDOWN_INCLUDES_DIRECTORY} "
141+ mkdir -p " ${report_include_directory} "
142+
143+ # Concatenate all deep dive reports as Markdown include
144+ rm -rf " ${report_include_directory} /AnomalyDetectionDeepDive.md"
145+ for markdown_file in $( find . -type f -name ' report_deep_dive_*.md' | sort) ; do
146+ cat " ${markdown_file} " >> " ${report_include_directory} /AnomalyDetectionDeepDive.md"
147+ echo " " >> " ${report_include_directory} /AnomalyDetectionDeepDive.md"
148+ rm -rf " ${markdown_file} "
149+ done
150+
151+ # Remove empty Markdown includes
152+ source " ${SCRIPTS_DIR} /cleanupAfterReportGeneration.sh" " ${report_include_directory} "
153+
154+ cp -f " ${ANOMALY_DETECTION_SUMMARY_DIR} /report.template.md" " ${FULL_REPORT_DIRECTORY} /report.template.md"
155+ cat " ${FULL_REPORT_DIRECTORY} /report.template.md" | " ${MARKDOWN_SCRIPTS_DIR} /embedMarkdownIncludes.sh" " ${report_include_directory} " > " ${FULL_REPORT_DIRECTORY} /anomaly_detection_report.md"
156+
157+ rm -rf " ${FULL_REPORT_DIRECTORY} /report.template.md"
158+ rm -rf " ${report_include_directory} "
159+ }
160+
161+ # Create report directory
162+ REPORT_NAME=" anomaly-detection"
163+ FULL_REPORT_DIRECTORY=" ${REPORTS_DIRECTORY} /${REPORT_NAME} "
164+ mkdir -p " ${FULL_REPORT_DIRECTORY} "
165+
166+ # Query Parameter key pairs for projection and algorithm side
167+ ALGORITHM_NODE=" projection_node_label"
168+ ALGORITHM_LANGUAGE=" projection_language"
169+ REPORT_NUMBER=" report_number"
170+
171+ # -- Overview Report for all code type -------------------------------
172+
173+ anomaly_detection_report_first_section
174+
175+ # -- Detail Reports for each code type -------------------------------
176+
177+ anomaly_detection_report " ${REPORT_NUMBER} =1" " ${ALGORITHM_NODE} =Artifact" " ${ALGORITHM_LANGUAGE} =Java"
178+ anomaly_detection_report " ${REPORT_NUMBER} =2" " ${ALGORITHM_NODE} =Package" " ${ALGORITHM_LANGUAGE} =Java"
179+ anomaly_detection_report " ${REPORT_NUMBER} =3" " ${ALGORITHM_NODE} =Type" " ${ALGORITHM_LANGUAGE} =Java"
180+ anomaly_detection_report " ${REPORT_NUMBER} =4" " ${ALGORITHM_NODE} =Module" " ${ALGORITHM_LANGUAGE} =Typescript"
181+
182+ # ---------------------------------------------------------------
183+
184+ anomaly_detection_finalize_report
185+
186+ echo " anomalyDetectionSummary: $( date +' %Y-%m-%dT%H:%M:%S%z' ) Successfully finished."
0 commit comments