Merge pull request #320 from JohT/feature/further-improve-logging

Further improve logging

Author: JohT

.github/workflows/internal-cypher-reference-documentation.yml

@@ -52,6 +52,7 @@ jobs:
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
+        set -x
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email "[email protected]"
         git fetch origin
@@ -62,4 +63,8 @@ jobs:
         git status
         git rebase --strategy-option=theirs origin/main --verbose
         git status
+        git add ./cypher/CYPHER.md        
+        git status
+        git commit --amend --no-edit
+        git status
         git push --verbose

.github/workflows/internal-environment-variables-reference-documentation.yml

@@ -52,6 +52,7 @@ jobs:
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
+        set -x
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email "[email protected]"
         git fetch origin
@@ -62,4 +63,8 @@ jobs:
         git status
         git rebase --strategy-option=theirs origin/main --verbose
         git status
-        git push --verbose
+        git add ./scripts/ENVIRONMENT_VARIABLES.md
+        git status
+        git commit --amend --no-edit
+        git status
+        git push --verbose

.github/workflows/internal-scripts-reference-documentation.yml

@@ -52,6 +52,7 @@ jobs:
       # And only run when the generated document changed to avoid an empty commit or an error while committing.
       if: github.event_name == 'push' && env.generated_document_changed
       run: |
+        set -x
         git config --global user.name '${{ env.CI_COMMIT_AUTHOR }}'
         git config --global user.email "[email protected]"
         git fetch origin
@@ -62,4 +63,8 @@ jobs:
         git status
         git rebase --strategy-option=theirs origin/main --verbose
         git status
-        git push --verbose
+        git add ./scripts/SCRIPTS.md
+        git status
+        git commit --amend --no-edit
+        git status
+        git push --verbose

scripts/analysis/analyze.sh

@@ -95,6 +95,7 @@ if [ ! -d "${ARTIFACTS_DIRECTORY}" ] && [ ! -d "${SOURCE_DIRECTORY}" ] ; then
     exit 1
 fi
 
+echo "${LOG_GROUP_START}Start Analysis"
 echo "analyze: analysisReportCompilation=${analysisReportCompilation}"
 echo "analyze: settingsProfile=${settingsProfile}"
 echo "analyze: exploreMode=${exploreMode}"
@@ -127,23 +128,24 @@ fi
 # Execute the settings profile script that sets all the necessary settings variables (overrideable by environment variables).
 echo "analyze: Using analysis settings profile script ${SETTINGS_PROFILE_SCRIPT}"
 source "${SETTINGS_PROFILE_SCRIPT}"
+echo "${LOG_GROUP_END}"
 
 # Setup Tools
-echo "${LOG_GROUP_START}Setup Tools";
+echo "${LOG_GROUP_START}Setup Tools"
 source "${SCRIPTS_DIR}/setupNeo4j.sh"
 source "${SCRIPTS_DIR}/setupJQAssistant.sh"
 source "${SCRIPTS_DIR}/startNeo4j.sh"
-echo "${LOG_GROUP_END}";
+echo "${LOG_GROUP_END}"
 
 # Scan and analyze artifacts when they were changed
-echo "${LOG_GROUP_START}Scan and Analyze Changed Artifacts";
+echo "${LOG_GROUP_START}Scan and Analyze Changed Artifacts"
 source "${SCRIPTS_DIR}/resetAndScanChanged.sh"
-echo "${LOG_GROUP_END}";
+echo "${LOG_GROUP_END}"
 
 # Prepare and validate graph database before analyzing and creating reports 
-echo "${LOG_GROUP_START}Prepare Analysis";
+echo "${LOG_GROUP_START}Prepare Analysis"
 source "${SCRIPTS_DIR}/prepareAnalysis.sh"
-echo "${LOG_GROUP_END}";
+echo "${LOG_GROUP_END}"
 
 if ${exploreMode}; then
   echo "analyze: Explore mode activated. Report generation will be skipped. Neo4j keeps running."
@@ -153,10 +155,9 @@ fi
 #########################
 # Create Reports
 #########################
-echo "analyze: Creating Reports with ${REPORT_COMPILATION_SCRIPT} ..."
 source "${REPORT_COMPILATION_SCRIPT}"
 
 # Stop Neo4j at the end
-echo "${LOG_GROUP_START}Finishing Analysis";
+echo "${LOG_GROUP_START}Finishing Analysis"
 source "${SCRIPTS_DIR}/stopNeo4j.sh"
-echo "${LOG_GROUP_END}";
+echo "${LOG_GROUP_END}"

scripts/reports/compilations/CsvReports.sh

@@ -24,9 +24,10 @@ echo "CsvReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 
 # Run all report scripts
 for report_script_file in "${REPORTS_SCRIPT_DIR}"/*Csv.sh; do
-    report_script_filename=$(basename -- "${report_script_file}")
-    
-    echo "${LOG_GROUP_START}${report_script_file}";
+    report_script_filename=$(basename -- "${report_script_file}");
+    report_script_filename="${report_script_filename%.*}" # Remove file extension
+
+    echo "${LOG_GROUP_START}Create CSV Report ${report_script_filename}";
     echo "CsvReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${report_script_filename}...";
 
     source "${report_script_file}"

scripts/reports/compilations/JupyterReports.sh

@@ -36,12 +36,13 @@ echo "JupyterReports: JUPYTER_NOTEBOOK_DIRECTORY=${JUPYTER_NOTEBOOK_DIRECTORY}"
 # Run all jupiter notebooks
 for jupyter_notebook_file in "${JUPYTER_NOTEBOOK_DIRECTORY}"/*.ipynb; do 
     jupyter_notebook_filename=$(basename -- "${jupyter_notebook_file}")
+    jupyter_notebook_filename_without_extension="${jupyter_notebook_filename%.*}" # Remove file extension
 
-    echo "${LOG_GROUP_START}${jupyter_notebook_filename}";
-    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${jupyter_notebook_filename}...";
+    echo "${LOG_GROUP_START}Create Jupyter Notebook Report ${jupyter_notebook_filename_without_extension}";
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${jupyter_notebook_filename_without_extension}...";
 
     source "${SCRIPTS_DIR}/executeJupyterNotebookReport.sh" --jupyterNotebook "${jupyter_notebook_filename}"
 
-    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${jupyter_notebook_filename}";
+    echo "JupyterReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Finished ${jupyter_notebook_filename_without_extension}";
     echo "${LOG_GROUP_END}";
 done

scripts/reports/compilations/VisualizationReports.sh

@@ -28,8 +28,9 @@ echo "VisualizationReports: REPORTS_SCRIPT_DIR=${REPORTS_SCRIPT_DIR}"
 # Run all visualization scripts
 for visualization_script_file in "${REPORTS_SCRIPT_DIR}"/*Visualization.sh; do 
     visualization_script_filename=$(basename -- "${visualization_script_file}")
-    
-    echo "${LOG_GROUP_START}${visualization_script_filename}";
+    visualization_script_filename="${visualization_script_filename%.*}" # Remove file extension
+
+    echo "${LOG_GROUP_START}Create Visualization Report ${visualization_script_filename}";
     echo "VisualizationReports: $(date +'%Y-%m-%dT%H:%M:%S%z') Starting ${visualization_script_filename}...";
 
     source "${visualization_script_file}"