Markdown table format support for Cypher queries

scripts/executeQuery.sh

@@ -14,9 +14,18 @@
 # -> "--no_source_reference" to not append the cypher query file name as last CSV column
 # -> any following key=value arguments are used as query parameters
 
+# Requires markdown/formatQueryResultAsMarkdownTable.sh
+
 # Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
 set -o errexit -o pipefail
 
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+SCRIPTS_DIR=${SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+#echo "executeQuery: SCRIPTS_DIR=$SCRIPTS_DIR" >&2
+
 # Overrideable Defaults
 NEO4J_HTTP_PORT=${NEO4J_HTTP_PORT:-"7474"} # Neo4j HTTP API port for executing queries
 NEO4J_HTTP_TRANSACTION_ENDPOINT=${NEO4J_HTTP_TRANSACTION_ENDPOINT:-"db/neo4j/tx/commit"} # Since Neo4j v5: "db/<name>/tx/commit", Neo4j v4: "db/data/transaction/commit"
@@ -35,29 +44,34 @@ fi
 cypher_query_file_name=""
 no_source_reference=false
 omit_query_error_highlighting=false
+output_markdown_table=false
 query_parameters=""
 
 # Input Arguments: Function to print usage information
 print_usage() {
-    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column] [--omit-query-error-highlighting]" >&2
+    echo "executeQuery Usage: $0 <filename> [--no-source-reference-column] [--omit-query-error-highlighting] [--output-markdown-table]" >&2
     echo "Options:" >&2
     echo "  --no-source-reference-column: Exclude the source reference column" >&2
     echo "  --omit-query-error-highlighting: Log query errors in same color as infos" >&2
+    echo "  --output-markdown-table: Output the result as markdown table instead of CSV" >&2
 }
 
 # Input Arguments: Parse the command-line arguments
 while [[ $# -gt 0 ]]; do
     arg="$1"
-
     case $arg in
         --no-source-reference-column)
             no_source_reference=true
             shift
             ;;
-        --omit_query_error_highlighting)
+        --omit-query-error-highlighting)
             omit_query_error_highlighting=true
             shift
             ;;
+        --output-markdown-table)
+            output_markdown_table=true
+            shift
+            ;;
         *)
             if [[ -z "${cypher_query_file_name}" ]]; then
                 # Input Arguments: Read the first unnamed input argument containing the name of the cypher file
@@ -134,11 +148,16 @@ if [[ -n "${error_message}" ]]; then
   exit 1
 fi
 
-# Output results in CSV format
-if [ "${no_source_reference}" = true ] ; then
-  echo -n "${cypher_query_result}" | jq -r '(.results[0])? | .columns,(.data[].row)? | map(if type == "array" then join(",") else . end) | flatten | @csv'
+if [ "${output_markdown_table}" = "true" ] ; then
+  echo "executeQuery: Will output in Markdown Table Format" >&2
+  echo -n "${cypher_query_result}" | "${SCRIPTS_DIR}/markdown/formatQueryResultAsMarkdownTable.sh"
 else
-  cypher_query_file_relative_name=${cypher_query_file_name#/**/cypher/}
-  sourceFileReferenceInfo="Source Cypher File: ${cypher_query_file_relative_name}"
-  echo -n "${cypher_query_result}" | jq -r --arg sourceReference "${sourceFileReferenceInfo}" '(.results[0])? | .columns + [$sourceReference], (.data[].row)? + [""]  | map(if type == "array" then join(",") else . end) | flatten | @csv'
+  # Output results in CSV format
+  if [ "${no_source_reference}" = true ] ; then
+    echo -n "${cypher_query_result}" | jq -r '(.results[0])? | .columns,(.data[].row)? | map(if type == "array" then join(",") else . end) | flatten | @csv'
+  else
+    cypher_query_file_relative_name=${cypher_query_file_name#/**/cypher/}
+    sourceFileReferenceInfo="Source Cypher File: ${cypher_query_file_relative_name}"
+    echo -n "${cypher_query_result}" | jq -r --arg sourceReference "${sourceFileReferenceInfo}" '(.results[0])? | .columns + [$sourceReference], (.data[].row)? + [""]  | map(if type == "array" then join(",") else . end) | flatten | @csv'
+  fi
 fi

scripts/markdown/embedMarkdownIncludes.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Processes a template_markdown_file markdown file, replacing placeholders like "<!-- include:intro.md -->" with the contents of the specified markdown files. The files to include needs to be in the "includes" subdirectory.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+#echo "embedMarkdownIncludes: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2
+
+template_markdown_file="$1"
+include_directory="includes"
+
+awk -v include_directory="${include_directory}" '
+  # Check if the filename is safe
+  function is_safe(path) {
+    if (substr(path, 1, 1) == "/") return 0
+    if (path ~ /\.\./) return 0
+    return 1
+  }
+
+  function include_file(path,   fullpath, line) {
+    fullpath = include_directory "/" path
+
+    if (!is_safe(path)) {
+      print "ERROR: illegal include path: " path > "/dev/stderr"
+      exit 1
+    }
+
+    if ((getline test < fullpath) < 0) {
+      print "ERROR: missing file " fullpath > "/dev/stderr"
+      exit 1
+    }
+    close(fullpath)
+
+    while ((getline line < fullpath) > 0) {
+      print line
+    }
+    close(fullpath)
+  }
+
+  {
+    # Look for the include marker using index+substr (portable)
+    if ($0 ~ /<!-- include:/) {
+      start = index($0, "include:") + 8
+      end   = index($0, "-->")
+      fname = substr($0, start, end - start)
+      gsub(/^[ \t]+|[ \t]+$/, "", fname)   # trim spaces
+
+      include_file(fname)
+    } else {
+      print
+    }
+  }
+' "${template_markdown_file}"
+
+#echo "embedMarkdownIncludes: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished." >&2

scripts/markdown/formatQueryResultAsMarkdownTable.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Takes the input stream (Cypher query result in JSON format) and formats it as a Markdown table.
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+#echo "formatQueryResultAsMarkdownTable: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2
+
+echo "formatQueryResultAsMarkdownTable: Will output in Markdown Table Format" >&2
+
+# Read all input (including multiline) into cypher_query_result
+cypher_query_result=$(cat)
+
+echo -n "${cypher_query_result}" | jq -r '
+  # Take the first query result
+  .results[0] as $result
+
+  # Extract the column names
+  | $result.columns as $columns
+
+  # Build the Markdown header row
+  | ( "| " + ( $columns | join(" | ") ) + " |" )
+
+  # Build the Markdown separator row
+  , ( "| " + ( $columns | map("---") | join(" | ") ) + " |" )
+
+  # Build one row for each data entry
+  , ( $result.data[].row
+      | map(tostring)
+      | "| " + ( join(" | ") ) + " |"
+    )
+'
+
+#echo "formatQueryResultAsMarkdownTable: $(date +'%Y-%m-%dT%H:%M:%S%z') Successfully finished." >&2

scripts/markdown/testEmbedMarkdownIncludes.sh

@@ -0,0 +1,95 @@
+#!/usr/bin/env bash
+
+# Tests template processing for markdown by embedding includes.
+
+# Requires embedMarkdownIncludes.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+echo "testEmbedMarkdownIncludes: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2
+
+tearDown() {
+  # echo "testEmbedMarkdownIncludes: Tear down tests...."
+  rm -rf "${temporaryTestDirectory}"
+}
+
+successful() {
+  local COLOR_SUCCESSFUL="\033[0;32m" # green 
+  local COLOR_DEFAULT='\033[0m'
+
+  echo -e "testEmbedMarkdownIncludes: ${COLOR_SUCCESSFUL}Tests finished successfully.${COLOR_DEFAULT}"
+
+  tearDown
+}
+
+fail() {
+  local COLOR_ERROR='\033[0;31m' # red
+  local COLOR_DEFAULT='\033[0m'
+
+  local errorMessage="${1}"
+
+  echo -e "testEmbedMarkdownIncludes: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}"
+  tearDown
+  return 1
+}
+
+echo "testEmbedMarkdownIncludes: Starting tests...."
+
+# Create testing resources
+temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory')
+
+testMarkdownTemplate="${temporaryTestDirectory}/testMarkdownTemplate.md"
+echo "<!-- include:testInclude.md -->" > "${testMarkdownTemplate}"
+
+# Setup test files
+mkdir -p "${temporaryTestDirectory}/includes"
+
+# ------------------------------------------------------------
+# Test case                                                 --
+# ------------------------------------------------------------
+echo "testEmbedMarkdownIncludes: 1.) An existing include file is correctly embedded."
+
+# - Setup
+testIncludeFile="includes/testInclude.md"
+expected_test_include_content="This is the included content for the test."
+echo "${expected_test_include_content}" > "${temporaryTestDirectory}/${testIncludeFile}"
+
+# - Execute script under test
+embeddedContent=$(cd "${temporaryTestDirectory}"; "${MARKDOWN_SCRIPTS_DIR}/embedMarkdownIncludes.sh" "${testMarkdownTemplate}" )
+
+# - Verify results
+if [ "${embeddedContent}" != "${expected_test_include_content}" ]; then
+  fail "1.) Test failed: Expected embedded content to be '${expected_test_include_content}', but got '${embeddedContent}'."
+fi
+
+# ------------------------------------------------------------
+# Test case                                                 --
+# ------------------------------------------------------------
+echo "testEmbedMarkdownIncludes: 2.) A missing include file results in an error."
+
+# - Setup
+testMarkdownTemplateMissingInclude="testMarkdownTemplateMissingInclude.md"
+echo "<!-- include:nonExistentFile.md -->" > "${temporaryTestDirectory}/${testMarkdownTemplateMissingInclude}"
+
+# - Execute script under test
+set +o errexit
+errorOutput=$(cd "${temporaryTestDirectory}"; { "${MARKDOWN_SCRIPTS_DIR}/embedMarkdownIncludes.sh" "${testMarkdownTemplateMissingInclude}" 2>&1 1>/dev/null; } )
+exitCode=$?
+set -o errexit
+
+# - Verify results
+if [ ${exitCode} -eq 0 ]; then
+  fail "2.) Test failed: Expected an error due to missing include file, but the script succeeded."
+fi
+if [[ "${errorOutput}" != *"ERROR: missing file"* ]]; then
+  fail "2.) Test failed: Expected error message to contain 'ERROR: missing file', but got '${errorOutput}'."
+fi
+
+successful
+return 0

scripts/markdown/testFormatQueryResultAsMarkdownTable.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+# Tests formatting of Cypher query results as Markdown table.
+
+# Requires formatQueryResultAsMarkdownTable.sh
+
+# Fail on any error ("-e" = exit on first error, "-o pipefail" exist on errors within piped commands)
+set -o errexit -o pipefail
+
+## Get this "scripts" directory if not already set
+# Even if $BASH_SOURCE is made for Bourne-like shells it is also supported by others and therefore here the preferred solution. 
+# CDPATH reduces the scope of the cd command to potentially prevent unintended directory changes.
+# This way non-standard tools like readlink aren't needed.
+MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR:-$( CDPATH=. cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P )} # Repository directory containing the shell scripts
+#echo "testFormatQueryResultAsMarkdownTable: MARKDOWN_SCRIPTS_DIR=${MARKDOWN_SCRIPTS_DIR}" >&2
+
+tearDown() {
+  # echo "testFormatQueryResultAsMarkdownTable: Tear down tests...."
+  rm -rf "${temporaryTestDirectory}"
+}
+
+successful() {
+  local COLOR_SUCCESSFUL="\033[0;32m" # green 
+  local COLOR_DEFAULT='\033[0m'
+
+  echo -e "testFormatQueryResultAsMarkdownTable: ${COLOR_SUCCESSFUL}Tests finished successfully.${COLOR_DEFAULT}"
+
+  tearDown
+}
+
+fail() {
+  local COLOR_ERROR='\033[0;31m' # red
+  local COLOR_DEFAULT='\033[0m'
+
+  local errorMessage="${1}"
+
+  echo -e "testFormatQueryResultAsMarkdownTable: ${COLOR_ERROR}${errorMessage}${COLOR_DEFAULT}"
+  tearDown
+  return 1
+}
+
+echo "testFormatQueryResultAsMarkdownTable: Starting tests...."
+
+# Create testing resources
+temporaryTestDirectory=$(mktemp -d 2>/dev/null || mktemp -d -t 'temporaryTestDirectory')
+
+# ------------------------------------------------------------
+# Test case                                                 --
+# ------------------------------------------------------------
+echo "testFormatQueryResultAsMarkdownTable: 1.) Convert a simple query result to a Markdown table."
+
+# Read expected result from test_data_cypher_query_result_simple_expected
+expected_result=$(<"${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_simple_expected.md")
+
+# - Execute script under test
+embeddedContent=$(cd "${temporaryTestDirectory}"; cat "${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_simple.json" | "${MARKDOWN_SCRIPTS_DIR}/formatQueryResultAsMarkdownTable.sh")
+
+# - Verify results
+if [ "${embeddedContent}" != "${expected_result}" ]; then
+  fail "1.) Test failed: Expected Markdown table to be \n${expected_result}, but got:\n${embeddedContent}"
+fi
+
+
+echo "testFormatQueryResultAsMarkdownTable: 2.) Convert an array query result to a Markdown table."
+
+# Read expected result from test_data_cypher_query_result_simple_expected
+expected_result=$(<"${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_with_array_expected.md")
+
+# - Execute script under test
+embeddedContent=$(cd "${temporaryTestDirectory}"; cat "${MARKDOWN_SCRIPTS_DIR}/test_data_cypher_query_result_with_array.json" | "${MARKDOWN_SCRIPTS_DIR}/formatQueryResultAsMarkdownTable.sh")
+
+# - Verify results
+if [ "${embeddedContent}" != "${expected_result}" ]; then
+  fail "2.) Test failed: Expected Markdown table to be \n${expected_result}, but got:\n${embeddedContent}"
+fi
+
+successful
+return 0

scripts/markdown/test_data_cypher_query_result_simple.json

@@ -0,0 +1,35 @@
+{
+    "results": [
+        {
+            "columns": [
+                "nodeLabel",
+                "nodesWithThatLabel",
+                "nodesWithThatLabelPercent"
+            ],
+            "data": [
+                {
+                    "row": [
+                        "Git",
+                        234151,
+                        77.73524646765112
+                    ]
+                },
+                {
+                    "row": [
+                        "Change",
+                        213807,
+                        70.98128917454584
+                    ]
+                },
+                {
+                    "row": [
+                        "Update",
+                        138781,
+                        46.073581748645495
+                    ]
+                }
+            ]
+        }
+    ],
+    "errors": []
+}