Add monitoring script

Author: drighetto

.github/workflows/monitoring-oshp-site-references.yml

@@ -0,0 +1,26 @@
+name: perform_monitoring_oshp_site_references
+on:
+  workflow_dispatch:
+  push:
+    paths:
+    - 'tab_casestudies.md'
+  schedule:
+    - cron: '0 0 * * 0'
+jobs:
+  build:
+    runs-on: ubuntu-latest   
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up Python 3.10
+      uses: actions/setup-python@v2
+      with:
+        python-version: "3.10"
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+    - name: Run the validation
+      run: |
+        cd ci; python monitoring_oshp_site_references.py
+
+

ci/monitoring_oshp_site_references.py

@@ -0,0 +1,64 @@
+"""
+Utility script to verify that every site mentioned in the tab named "Case Studies" have a mention to OSHP.
+
+The goal is to allow detection of site not mentioning the OSHP anymore and then update the "Case Studies" content.
+
+Try to not use any external dependency to stay the more portable possible.
+"""
+import re
+import urllib.request
+
+OSHP_MARKER_STRINGS = ["owasp secure headers project", "https://owasp.org/www-project-secure-headers", "https://www.owasp.org/index.php/security_headers", "secure headers project"]
+DEFAULT_ENCODING = "utf-8"
+IGNORED_HTTP_RESPONSE_CODES = [401]
+SOURCE_MD_FILE = "../tab_casestudies.md"
+USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"
+
+
+def verify_mention(site_url):
+    oshp_is_mentioned = "NO"
+    request = urllib.request.Request(url=site_url, method="GET")
+    request.add_header("User-Agent", USER_AGENT)
+    request.add_header("Accept", "*/*")
+    request.add_header("Accept-Encoding", "deflate")
+    try:
+        # Assume by default that site refer to static non SPA page
+        with urllib.request.urlopen(request) as f:
+            content = f.read().decode(DEFAULT_ENCODING)
+        content_lower = content.lower()
+        for marker in OSHP_MARKER_STRINGS:
+            if marker in content_lower:
+                oshp_is_mentioned = "YES"
+                break
+        # If mention is not detected then try to check if it's an SPA
+        if oshp_is_mentioned == "NO":
+            expr = r'app\.[a-f0-9]+\.js'
+            bundles = re.findall(expr, content)
+            if len(bundles) > 0 or "React" in content:
+                oshp_is_mentioned = "NO => SPA"
+    except urllib.error.HTTPError as e:
+        if e.code in IGNORED_HTTP_RESPONSE_CODES:
+            oshp_is_mentioned = f"HTTP {e.code}"
+    return oshp_is_mentioned
+
+
+def extract_site_urls():
+    expr = r'\*\s+\[[a-zA-Z0-9\s_\-\.]+\]\((.*?)\)'
+    with open(SOURCE_MD_FILE, mode="r", encoding=DEFAULT_ENCODING) as f:
+        content = f.read()
+    return re.findall(expr, content)
+
+
+if __name__ == "__main__":
+    verify_mention("https://eoahgl4pezvrojk.m.pipedream.net/")
+    print("[+] Extract site urls...")
+    site_urls = extract_site_urls()
+    print(f"{len(site_urls)} urls founds.")
+    print("[+] Verify mention to OSHP on each site...")
+    lines = []
+    for site_url in site_urls:
+        if site_url.strip().startswith("http"):
+            oshp_is_mentioned = verify_mention(site_url)
+            lines.append(f"[{str(oshp_is_mentioned):<9}] {site_url}")
+    lines.sort()
+    print("\n".join(lines))

tab_casestudies.md

@@ -44,11 +44,10 @@ https://owasp.org/www-project-secure-headers/index.html#div-technical_php
 
 * [Cloud.gov](https://cloud.gov/docs/management/headers/).
 * [Amazon AWS](https://docs.aws.amazon.com/whitepapers/latest/secure-content-delivery-amazon-cloudfront/improving-security-by-enabling-security-specific-headers.html).
-* [Salesforce](https://documentation.b2c.commercecloud.salesforce.com/DOC1/topic/com.demandware.dochelp/content/b2c_commerce/topics/b2c_security_best_practices/b2c_declarative_security_via_http_headers.html?resultof=%22%68%65%61%64%65%72%73%22%20%22%68%65%61%64%65%72%22%20).
+* [Salesforce](https://help.salesforce.com/s/articleView?language=en_US&id=cc.b2c_declarative_security_via_http_headers.htm&type=5).
 * [Black Hills Information Security](https://www.blackhillsinfosec.com/fixing-content-security-policies-with-cloudflare-workers/).
 * [Progress](https://www.progress.com/documentation/sitefinity-cms/110/predefined-security-headers-in-http-response).
 * [Bloomreach](https://documentation.bloomreach.com/14/library/concepts/security/configure-security-response-headers.html).
-* [CrashTest Security](https://crashtest-security.com/enable-security-headers/).
 * [Tableau](https://help.tableau.com/current/server-linux/en-us/security_http_headers.htm).
 * [42Crunch](https://docs.42crunch.com/latest/content/extras/protection_security_headers.htm).
 * [SAP](https://help.sap.com/docs/SAP_UPSCALE_COMMERCE/4620dd88ff9047c89ffb7fa897207a46/30af09ca9e394505a85661fa530d1263.html).